X-Git-Url: https://git.ktnx.net/?p=mobile-ledger.git;a=blobdiff_plain;f=app%2Fsrc%2Fmain%2Fjava%2Fnet%2Fktnx%2Fmobileledger%2Fasync%2FRetrieveTransactionsTask.java;h=8ac793663eb686ccc00d428db93b965ef8c4a2f8;hp=ac9ec5093e54ece20790e6107f8fe2d593c5d9af;hb=24eea00d6f49faeaf5bbbdad6178dc53ecfa08a5;hpb=1930f3fa34616229bd6a83654e6527672064196a diff --git a/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java b/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java index ac9ec509..8ac79366 100644 --- a/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java +++ b/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java @@ -1,5 +1,5 @@ /* - * Copyright © 2020 Damyan Ivanov. + * Copyright © 2021 Damyan Ivanov. * This file is part of MoLe. * MoLe is free software: you can distribute it and/or modify it * under the term of the GNU General Public License as published by @@ -23,19 +23,34 @@ import android.os.AsyncTask; import android.os.OperationCanceledException; import androidx.annotation.NonNull; +import androidx.room.Transaction; + +import com.fasterxml.jackson.databind.RuntimeJsonMappingException; import net.ktnx.mobileledger.App; +import net.ktnx.mobileledger.dao.AccountDAO; +import net.ktnx.mobileledger.dao.AccountValueDAO; +import net.ktnx.mobileledger.dao.TransactionAccountDAO; +import net.ktnx.mobileledger.dao.TransactionDAO; +import net.ktnx.mobileledger.db.Account; +import net.ktnx.mobileledger.db.AccountWithAmounts; +import net.ktnx.mobileledger.db.DB; +import net.ktnx.mobileledger.db.Option; +import net.ktnx.mobileledger.db.Profile; +import net.ktnx.mobileledger.db.TransactionAccount; +import net.ktnx.mobileledger.db.TransactionWithAccounts; import net.ktnx.mobileledger.err.HTTPException; -import net.ktnx.mobileledger.json.v1_15.AccountListParser; -import net.ktnx.mobileledger.json.v1_15.ParsedBalance; -import net.ktnx.mobileledger.json.v1_15.ParsedLedgerAccount; -import net.ktnx.mobileledger.json.v1_15.ParsedLedgerTransaction; -import net.ktnx.mobileledger.json.v1_15.TransactionListParser; +import net.ktnx.mobileledger.json.API; +import net.ktnx.mobileledger.json.AccountListParser; +import net.ktnx.mobileledger.json.ApiNotSupportedException; +import net.ktnx.mobileledger.json.TransactionListParser; import net.ktnx.mobileledger.model.Data; import net.ktnx.mobileledger.model.LedgerAccount; import net.ktnx.mobileledger.model.LedgerTransaction; import net.ktnx.mobileledger.model.LedgerTransactionAccount; -import net.ktnx.mobileledger.model.MobileLedgerProfile; +import net.ktnx.mobileledger.ui.MainModel; +import net.ktnx.mobileledger.utils.Logger; +import net.ktnx.mobileledger.utils.MLDB; import net.ktnx.mobileledger.utils.NetworkUtil; import java.io.BufferedReader; @@ -48,15 +63,18 @@ import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; -public class RetrieveTransactionsTask - extends AsyncTask { +public class RetrieveTransactionsTask extends + AsyncTask { private static final int MATCHING_TRANSACTIONS_LIMIT = 150; private static final Pattern reComment = Pattern.compile("^\\s*;"); private static final Pattern reTransactionStart = Pattern.compile( @@ -71,12 +89,15 @@ public class RetrieveTransactionsTask private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$"); private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$"); // %3A is '=' - private Pattern reAccountName = Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\""); - private Pattern reAccountValue = Pattern.compile( + private final Pattern reAccountName = + Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\""); + private final Pattern reAccountValue = Pattern.compile( "\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?"); - private MobileLedgerProfile profile; + private final MainModel mainModel; + private final Profile profile; private int expectedPostingsCount = -1; - public RetrieveTransactionsTask(@NonNull MobileLedgerProfile profile) { + public RetrieveTransactionsTask(@NonNull MainModel mainModel, @NonNull Profile profile) { + this.mainModel = mainModel; this.profile = profile; } private static void L(String msg) { @@ -115,11 +136,11 @@ public class RetrieveTransactionsTask Data.backgroundTaskProgress.postValue(values[0]); } @Override - protected void onPostExecute(String error) { - super.onPostExecute(error); + protected void onPostExecute(Result result) { + super.onPostExecute(result); Progress progress = new Progress(); progress.setState(ProgressState.FINISHED); - progress.setError(error); + progress.setError(result.error); onProgressUpdate(progress); } @Override @@ -129,15 +150,14 @@ public class RetrieveTransactionsTask progress.setState(ProgressState.FINISHED); onProgressUpdate(progress); } - private String retrieveTransactionListLegacy() throws IOException, HTTPException { + private void retrieveTransactionListLegacy(List accounts, + List transactions) + throws IOException, HTTPException { Progress progress = Progress.indeterminate(); progress.setState(ProgressState.RUNNING); progress.setTotal(expectedPostingsCount); int maxTransactionId = -1; - ArrayList list = new ArrayList<>(); HashMap map = new HashMap<>(); - ArrayList displayed = new ArrayList<>(); - ArrayList transactions = new ArrayList<>(); LedgerAccount lastAccount = null; ArrayList syntheticAccounts = new ArrayList<>(); @@ -200,9 +220,9 @@ public class RetrieveTransactionsTask else { parentAccount = null; } - lastAccount = new LedgerAccount(profile, accName, parentAccount); + lastAccount = new LedgerAccount(accName, parentAccount); - list.add(lastAccount); + accounts.add(lastAccount); map.put(accName, lastAccount); state = ParserState.EXPECTING_ACCOUNT_AMOUNT; @@ -284,7 +304,7 @@ public class RetrieveTransactionsTask if (m.find()) { if (transactionId == 0) throw new TransactionParserException( - "Transaction Id is 0 while expecting " + "description"); + "Transaction Id is 0 while expecting description"); String date = Objects.requireNonNull(m.group(1)); try { @@ -295,8 +315,8 @@ public class RetrieveTransactionsTask new LedgerTransaction(transactionId, date, m.group(2)); } catch (ParseException e) { - e.printStackTrace(); - return String.format("Error parsing date '%s'", date); + throw new TransactionParserException( + String.format("Error parsing date '%s'", date)); } state = ParserState.EXPECTING_TRANSACTION_DETAILS; L(String.format(Locale.ENGLISH, @@ -314,7 +334,7 @@ public class RetrieveTransactionsTask state = ParserState.EXPECTING_TRANSACTION; L(String.format("transaction %s parsed → expecting transaction", - transaction.getId())); + transaction.getLedgerId())); // sounds like a good idea, but transaction-1 may not be the first one chronologically // for example, when you add the initial seeding transaction after entering some others @@ -329,8 +349,9 @@ public class RetrieveTransactionsTask LedgerTransactionAccount lta = parseTransactionAccountLine(line); if (lta != null) { transaction.addAccount(lta); - L(String.format(Locale.ENGLISH, "%d: %s = %s", transaction.getId(), - lta.getAccountName(), lta.getAmount())); + L(String.format(Locale.ENGLISH, "%d: %s = %s", + transaction.getLedgerId(), lta.getAccountName(), + lta.getAmount())); } else throw new IllegalStateException( @@ -345,15 +366,11 @@ public class RetrieveTransactionsTask } throwIfCancelled(); - - profile.setAndStoreAccountAndTransactionListFromWeb(list, transactions); - - return null; } } - private @NonNull - LedgerAccount ensureAccountExists(String accountName, HashMap map, - ArrayList createdAccounts) { + @NonNull + public LedgerAccount ensureAccountExists(String accountName, HashMap map, + ArrayList createdAccounts) { LedgerAccount acc = map.get(accountName); if (acc != null) @@ -368,18 +385,52 @@ public class RetrieveTransactionsTask parentAccount = null; } - acc = new LedgerAccount(profile, accountName, parentAccount); + acc = new LedgerAccount(accountName, parentAccount); createdAccounts.add(acc); return acc; } - private boolean retrieveAccountList() throws IOException, HTTPException { + public void addNumberOfPostings(int number) { + expectedPostingsCount += number; + } + private List retrieveAccountList() + throws IOException, HTTPException, ApiNotSupportedException { + final API apiVersion = API.valueOf(profile.getApiVersion()); + if (apiVersion.equals(API.auto)) { + return retrieveAccountListAnyVersion(); + } + else if (apiVersion.equals(API.html)) { + Logger.debug("json", + "Declining using JSON API for /accounts with configured legacy API version"); + return null; + } + else { + return retrieveAccountListForVersion(apiVersion); + } + } + private List retrieveAccountListAnyVersion() throws ApiNotSupportedException { + for (API ver : API.allVersions) { + try { + return retrieveAccountListForVersion(ver); + } + catch (Exception e) { + Logger.debug("json", + String.format(Locale.US, "Error during account list retrieval using API %s", + ver.getDescription())); + } + + } + + throw new ApiNotSupportedException(); + } + private List retrieveAccountListForVersion(API version) + throws IOException, HTTPException { HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts"); http.setAllowUserInteraction(false); switch (http.getResponseCode()) { case 200: break; case 404: - return false; + return null; default: throw new HTTPException(http.getResponseCode(), http.getResponseMessage()); } @@ -387,84 +438,62 @@ public class RetrieveTransactionsTask SQLiteDatabase db = App.getDatabase(); ArrayList list = new ArrayList<>(); HashMap map = new HashMap<>(); - HashMap currentMap = new HashMap<>(); - for (LedgerAccount acc : Objects.requireNonNull(profile.getAllAccounts())) - currentMap.put(acc.getName(), acc); + throwIfCancelled(); try (InputStream resp = http.getInputStream()) { + throwIfCancelled(); if (http.getResponseCode() != 200) throw new IOException(String.format("HTTP error %d", http.getResponseCode())); - AccountListParser parser = new AccountListParser(resp); + AccountListParser parser = AccountListParser.forApiVersion(version, resp); expectedPostingsCount = 0; while (true) { throwIfCancelled(); - ParsedLedgerAccount parsedAccount = parser.nextAccount(); - if (parsedAccount == null) { + LedgerAccount acc = parser.nextAccount(this, map); + if (acc == null) break; - } - expectedPostingsCount += parsedAccount.getAnumpostings(); - final String accName = parsedAccount.getAname(); - LedgerAccount acc = map.get(accName); - if (acc != null) - throw new RuntimeException( - String.format("Account '%s' already present", acc.getName())); - String parentName = LedgerAccount.extractParentName(accName); - ArrayList createdParents = new ArrayList<>(); - LedgerAccount parent; - if (parentName == null) { - parent = null; - } - else { - parent = ensureAccountExists(parentName, map, createdParents); - parent.setHasSubAccounts(true); - } - acc = new LedgerAccount(profile, accName, parent); list.add(acc); - map.put(accName, acc); - - String lastCurrency = null; - float lastCurrencyAmount = 0; - for (ParsedBalance b : parsedAccount.getAibalance()) { - throwIfCancelled(); - final String currency = b.getAcommodity(); - final float amount = b.getAquantity() - .asFloat(); - if (currency.equals(lastCurrency)) { - lastCurrencyAmount += amount; - } - else { - if (lastCurrency != null) { - acc.addAmount(lastCurrencyAmount, lastCurrency); - } - lastCurrency = currency; - lastCurrencyAmount = amount; - } - } - if (lastCurrency != null) { - acc.addAmount(lastCurrencyAmount, lastCurrency); - } - for (LedgerAccount p : createdParents) - acc.propagateAmountsTo(p); } throwIfCancelled(); } - // the current account tree may have changed, update the new-to be tree to match - for (LedgerAccount acc : list) { - LedgerAccount prevData = currentMap.get(acc.getName()); - if (prevData != null) { - acc.setExpanded(prevData.isExpanded()); - acc.setAmountsExpanded(prevData.amountsExpanded()); + return list; + } + private List retrieveTransactionList() + throws ParseException, HTTPException, IOException, ApiNotSupportedException { + final API apiVersion = API.valueOf(profile.getApiVersion()); + if (apiVersion.equals(API.auto)) { + return retrieveTransactionListAnyVersion(); + } + else if (apiVersion.equals(API.html)) { + Logger.debug("json", + "Declining using JSON API for /accounts with configured legacy API version"); + return null; + } + else { + return retrieveTransactionListForVersion(apiVersion); + } + + } + private List retrieveTransactionListAnyVersion() + throws ApiNotSupportedException { + for (API ver : API.allVersions) { + try { + return retrieveTransactionListForVersion(ver); } + catch (Exception e) { + Logger.debug("json", + String.format(Locale.US, "Error during account list retrieval using API %s", + ver.getDescription())); + } + } - profile.setAndStoreAccountListFromWeb(list); - return true; + throw new ApiNotSupportedException(); } - private boolean retrieveTransactionList() throws IOException, ParseException, HTTPException { + private List retrieveTransactionListForVersion(API apiVersion) + throws IOException, ParseException, HTTPException { Progress progress = new Progress(); - int maxTransactionId = Data.transactions.size(); progress.setTotal(expectedPostingsCount); HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions"); @@ -474,74 +503,167 @@ public class RetrieveTransactionsTask case 200: break; case 404: - return false; + return null; default: throw new HTTPException(http.getResponseCode(), http.getResponseMessage()); } + ArrayList trList = new ArrayList<>(); try (InputStream resp = http.getInputStream()) { throwIfCancelled(); - ArrayList trList = new ArrayList<>(); - TransactionListParser parser = new TransactionListParser(resp); + TransactionListParser parser = TransactionListParser.forApiVersion(apiVersion, resp); int processedPostings = 0; while (true) { throwIfCancelled(); - ParsedLedgerTransaction parsedTransaction = parser.nextTransaction(); + LedgerTransaction transaction = parser.nextTransaction(); throwIfCancelled(); - if (parsedTransaction == null) + if (transaction == null) break; - LedgerTransaction transaction = parsedTransaction.asLedgerTransaction(); trList.add(transaction); progress.setProgress(processedPostings += transaction.getAccounts() .size()); +// Logger.debug("trParser", +// String.format(Locale.US, "Parsed transaction %d - %s", transaction +// .getId(), +// transaction.getDescription())); +// for (LedgerTransactionAccount acc : transaction.getAccounts()) { +// Logger.debug("trParser", +// String.format(Locale.US, " %s", acc.getAccountName())); +// } publishProgress(progress); } throwIfCancelled(); - profile.setAndStoreTransactionList(trList); } - return true; + // json interface returns transactions if file order and the rest of the machinery + // expects them in reverse chronological order + Collections.sort(trList, (o1, o2) -> { + int res = o2.getDate() + .compareTo(o1.getDate()); + if (res != 0) + return res; + return Long.compare(o2.getLedgerId(), o1.getLedgerId()); + }); + return trList; } @SuppressLint("DefaultLocale") @Override - protected String doInBackground(Void... params) { + protected Result doInBackground(Void... params) { Data.backgroundTaskStarted(); + List accounts; + List transactions; try { - if (!retrieveAccountList() || !retrieveTransactionList()) - return retrieveTransactionListLegacy(); - return null; + accounts = retrieveAccountList(); + // accounts is null in API-version auto-detection and means + // requesting 'html' API version via the JSON classes + // this can't work, and the null results in the legacy code below + // being called + if (accounts == null) + transactions = null; + else + transactions = retrieveTransactionList(); + + if (accounts == null || transactions == null) { + accounts = new ArrayList<>(); + transactions = new ArrayList<>(); + retrieveTransactionListLegacy(accounts, transactions); + } + + storeAccountsAndTransactions(accounts, transactions); + + mainModel.updateDisplayedTransactionsFromWeb(transactions); + + return new Result(accounts, transactions); } catch (MalformedURLException e) { e.printStackTrace(); - return "Invalid server URL"; + return new Result("Invalid server URL"); } catch (HTTPException e) { e.printStackTrace(); - return String.format("HTTP error %d: %s", e.getResponseCode(), e.getResponseMessage()); + return new Result( + String.format("HTTP error %d: %s", e.getResponseCode(), e.getMessage())); } catch (IOException e) { e.printStackTrace(); - return e.getLocalizedMessage(); + return new Result(e.getLocalizedMessage()); + } + catch (RuntimeJsonMappingException e) { + e.printStackTrace(); + return new Result(Result.ERR_JSON_PARSER_ERROR); } catch (ParseException e) { e.printStackTrace(); - return "Network error"; + return new Result("Network error"); } catch (OperationCanceledException e) { e.printStackTrace(); - return "Operation cancelled"; + return new Result("Operation cancelled"); + } + catch (ApiNotSupportedException e) { + e.printStackTrace(); + return new Result("Server version not supported"); } finally { Data.backgroundTaskFinished(); } } - private void throwIfCancelled() { + @Transaction + private void storeAccountsAndTransactions(List accounts, + List transactions) { + AccountDAO accDao = DB.get() + .getAccountDAO(); + TransactionDAO trDao = DB.get() + .getTransactionDAO(); + TransactionAccountDAO trAccDao = DB.get() + .getTransactionAccountDAO(); + AccountValueDAO valDao = DB.get() + .getAccountValueDAO(); + + final List list = new ArrayList<>(); + for (LedgerAccount acc : accounts) { + final AccountWithAmounts a = acc.toDBOWithAmounts(); + Account existing = accDao.getByNameSync(profile.getId(), acc.getName()); + if (existing != null) { + a.account.setExpanded(existing.isExpanded()); + a.account.setAmountsExpanded(existing.isAmountsExpanded()); + a.account.setId( + existing.getId()); // not strictly needed, but since we have it anyway... + } + + list.add(a); + } + accDao.storeAccountsSync(list, profile.getId()); + + long trGen = trDao.getGenerationSync(profile.getId()); + for (LedgerTransaction tr : transactions) { + TransactionWithAccounts tran = tr.toDBO(); + tran.transaction.setGeneration(trGen); + tran.transaction.setProfileId(profile.getId()); + + tran.transaction.setId(trDao.insertSync(tran.transaction)); + + for (TransactionAccount trAcc : tran.accounts) { + trAcc.setGeneration(trGen); + trAcc.setTransactionId(tran.transaction.getId()); + trAcc.setId(trAccDao.insertSync(trAcc)); + } + } + + trDao.purgeOldTransactionsSync(profile.getId(), trGen); + + DB.get() + .getOptionDAO() + .insertSync(new Option(profile.getId(), MLDB.OPT_LAST_SCRAPE, + String.valueOf((new Date()).getTime()))); + } + public void throwIfCancelled() { if (isCancelled()) throw new OperationCanceledException(null); } @@ -624,4 +746,18 @@ public class RetrieveTransactionsTask super(message); } } + + public static class Result { + public static String ERR_JSON_PARSER_ERROR = "err_json_parser"; + public String error; + public List accounts; + public List transactions; + Result(String error) { + this.error = error; + } + Result(List accounts, List transactions) { + this.accounts = accounts; + this.transactions = transactions; + } + } }