X-Git-Url: https://git.ktnx.net/?p=mobile-ledger.git;a=blobdiff_plain;f=app%2Fsrc%2Fmain%2Fjava%2Fnet%2Fktnx%2Fmobileledger%2Fasync%2FRetrieveTransactionsTask.java;h=8d98dc63c89e920ce1128df939f0ee014b33805b;hp=c1e51442c781e12bbf8f801edcae873947a7307c;hb=23b8360d4f884bbdb20cc2de1825486ef42f37d6;hpb=73596d7b17563d85a94889a1b5d3b9450d115765 diff --git a/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java b/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java index c1e51442..8d98dc63 100644 --- a/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java +++ b/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java @@ -50,14 +50,17 @@ import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Locale; import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static net.ktnx.mobileledger.utils.Logger.debug; + public class RetrieveTransactionsTask extends AsyncTask { - private static final int MATCHING_TRANSACTIONS_LIMIT = 50; + private static final int MATCHING_TRANSACTIONS_LIMIT = 150; private static final Pattern reComment = Pattern.compile("^\\s*;"); private static final Pattern reTransactionStart = Pattern.compile("([\\d.-]+)"); @@ -76,7 +79,7 @@ public class RetrieveTransactionsTask this.contextRef = contextRef; } private static void L(String msg) { - //Log.d("transaction-parser", msg); + //debug("transaction-parser", msg); } @Override protected void onProgressUpdate(Progress... values) { @@ -107,18 +110,25 @@ public class RetrieveTransactionsTask context.onRetrieveDone(null); } private String retrieveTransactionListLegacy(MobileLedgerProfile profile) - throws IOException, ParseException { + throws IOException, ParseException, HTTPException { Progress progress = new Progress(); int maxTransactionId = Progress.INDETERMINATE; ArrayList accountList = new ArrayList<>(); HashMap accountNames = new HashMap<>(); - LedgerAccount lastAccount = null; + HashMap syntheticAccounts = new HashMap<>(); + LedgerAccount lastAccount = null, prevAccount = null; boolean onlyStarred = Data.optShowOnlyStarred.get(); HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal"); http.setAllowUserInteraction(false); publishProgress(progress); - try (SQLiteDatabase db = MLDB.getWritableDatabase()) { + switch (http.getResponseCode()) { + case 200: + break; + default: + throw new HTTPException(http.getResponseCode(), http.getResponseMessage()); + } + try (SQLiteDatabase db = MLDB.getDatabase()) { try (InputStream resp = http.getInputStream()) { if (http.getResponseCode() != 200) throw new IOException(String.format("HTTP error %d", http.getResponseCode())); @@ -153,7 +163,13 @@ public class RetrieveTransactionsTask if (line.equals("

General Journal

")) { state = ParserState.EXPECTING_TRANSACTION; L("→ expecting transaction"); - Data.accounts.set(accountList); + // commit the current transaction and start a new one + // the account list in the UI should reflect the (committed) + // state of the database + db.setTransactionSuccessful(); + db.endTransaction(); + Data.accounts.setList(accountList); + db.beginTransaction(); continue; } m = reAccountName.matcher(line); @@ -163,14 +179,20 @@ public class RetrieveTransactionsTask acct_name = acct_name.replace("\"", ""); L(String.format("found account: %s", acct_name)); - lastAccount = profile.loadAccount(acct_name); - if (lastAccount == null) { + prevAccount = lastAccount; + lastAccount = profile.tryLoadAccount(db, acct_name); + if (lastAccount == null) lastAccount = new LedgerAccount(acct_name); - profile.storeAccount(db, lastAccount); - } + else lastAccount.removeAmounts(); + profile.storeAccount(db, lastAccount); + if (prevAccount != null) prevAccount + .setHasSubAccounts(prevAccount.isParentOf(lastAccount)); // make sure the parent account(s) are present, // synthesising them if necessary + // this happens when the (missing-in-HTML) parent account has + // only one child so we create a synthetic parent account record, + // copying the amounts when child's amounts are parsed String parentName = lastAccount.getParentName(); if (parentName != null) { Stack toAppend = new Stack<>(); @@ -180,19 +202,29 @@ public class RetrieveTransactionsTask parentName = new LedgerAccount(parentName).getParentName(); } + syntheticAccounts.clear(); while (!toAppend.isEmpty()) { String aName = toAppend.pop(); - LedgerAccount acc = new LedgerAccount(aName); - acc.setHidden(lastAccount.isHidden()); - if (!onlyStarred || !acc.isHidden()) - accountList.add(acc); + LedgerAccount acc = profile.tryLoadAccount(db, aName); + if (acc == null) { + acc = new LedgerAccount(aName); + acc.setHiddenByStar(lastAccount.isHiddenByStar()); + acc.setExpanded(!lastAccount.hasSubAccounts() || + lastAccount.isExpanded()); + } + acc.setHasSubAccounts(true); + acc.removeAmounts(); // filled below when amounts are parsed + if ((!onlyStarred || !acc.isHiddenByStar()) && + acc.isVisible(accountList)) accountList.add(acc); L(String.format("gap-filling with %s", aName)); accountNames.put(aName, null); profile.storeAccount(db, acc); + syntheticAccounts.put(aName, acc); } } - if (!onlyStarred || !lastAccount.isHidden()) + if ((!onlyStarred || !lastAccount.isHiddenByStar()) && + lastAccount.isVisible(accountList)) accountList.add(lastAccount); accountNames.put(acct_name, null); @@ -213,9 +245,14 @@ public class RetrieveTransactionsTask if (currency == null) currency = ""; value = value.replace(',', '.'); L("curr=" + currency + ", value=" + value); + final float val = Float.parseFloat(value); profile.storeAccountValue(db, lastAccount.getName(), currency, - Float.valueOf(value)); - lastAccount.addAmount(Float.parseFloat(value), currency); + val); + lastAccount.addAmount(val, currency); + for (LedgerAccount syn : syntheticAccounts.values()) { + syn.addAmount(val, currency); + profile.storeAccountValue(db, syn.getName(), currency, val); + } } if (match_found) { @@ -231,7 +268,8 @@ public class RetrieveTransactionsTask if (m.find()) { transactionId = Integer.valueOf(m.group(1)); state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION; - L(String.format("found transaction %d → expecting description", + L(String.format(Locale.ENGLISH, + "found transaction %d → expecting description", transactionId)); progress.setProgress(++processedTransactionCount); if (maxTransactionId < transactionId) @@ -268,9 +306,9 @@ public class RetrieveTransactionsTask return String.format("Error parsing date '%s'", date); } state = ParserState.EXPECTING_TRANSACTION_DETAILS; - L(String.format("transaction %d created for %s (%s) →" + - " expecting details", transactionId, date, - m.group(2))); + L(String.format(Locale.ENGLISH, + "transaction %d created for %s (%s) →" + + " expecting details", transactionId, date, m.group(2))); } break; @@ -321,8 +359,8 @@ public class RetrieveTransactionsTask transaction.addAccount( new LedgerTransactionAccount(acc_name, Float.valueOf(amount), currency)); - L(String.format("%d: %s = %s", transaction.getId(), - acc_name, amount)); + L(String.format(Locale.ENGLISH, "%d: %s = %s", + transaction.getId(), acc_name, amount)); } else throw new IllegalStateException(String.format( "Can't parse transaction %d " + "details: %s", @@ -372,44 +410,76 @@ public class RetrieveTransactionsTask throw new HTTPException(http.getResponseCode(), http.getResponseMessage()); } publishProgress(progress); - try (SQLiteDatabase db = MLDB.getWritableDatabase()) { - try (InputStream resp = http.getInputStream()) { - if (http.getResponseCode() != 200) - throw new IOException(String.format("HTTP error %d", http.getResponseCode())); + SQLiteDatabase db = MLDB.getDatabase(); + ArrayList accountList = new ArrayList<>(); + boolean listFilledOK = false; + try (InputStream resp = http.getInputStream()) { + if (http.getResponseCode() != 200) + throw new IOException(String.format("HTTP error %d", http.getResponseCode())); - db.beginTransaction(); - try { - profile.markAccountsAsNotPresent(db); + db.beginTransaction(); + try { + profile.markAccountsAsNotPresent(db); - AccountListParser parser = new AccountListParser(resp); - ArrayList accountList = new ArrayList<>(); + AccountListParser parser = new AccountListParser(resp); - while (true) { - throwIfCancelled(); - ParsedLedgerAccount parsedAccount = parser.nextAccount(); - if (parsedAccount == null) break; - - LedgerAccount acc = new LedgerAccount(parsedAccount.getAname()); - profile.storeAccount(db, acc); - for (ParsedBalance b : parsedAccount.getAebalance()) { - profile.storeAccountValue(db, acc.getName(), b.getAcommodity(), - b.getAquantity().asFloat()); + LedgerAccount prevAccount = null; + + while (true) { + throwIfCancelled(); + ParsedLedgerAccount parsedAccount = parser.nextAccount(); + if (parsedAccount == null) break; + + LedgerAccount acc = profile.tryLoadAccount(db, parsedAccount.getAname()); + if (acc == null) acc = new LedgerAccount(parsedAccount.getAname()); + else acc.removeAmounts(); + + profile.storeAccount(db, acc); + String lastCurrency = null; + float lastCurrencyAmount = 0; + for (ParsedBalance b : parsedAccount.getAibalance()) { + final String currency = b.getAcommodity(); + final float amount = b.getAquantity().asFloat(); + if (currency.equals(lastCurrency)) lastCurrencyAmount += amount; + else { + if (lastCurrency != null) { + profile.storeAccountValue(db, acc.getName(), lastCurrency, + lastCurrencyAmount); + acc.addAmount(lastCurrencyAmount, lastCurrency); + } + lastCurrency = currency; + lastCurrencyAmount = amount; } + } + if (lastCurrency != null) { + profile.storeAccountValue(db, acc.getName(), lastCurrency, + lastCurrencyAmount); + acc.addAmount(lastCurrencyAmount, lastCurrency); + } + + if (acc.isVisible(accountList)) accountList.add(acc); - accountList.add(acc); + if (prevAccount != null) { + prevAccount.setHasSubAccounts( + acc.getName().startsWith(prevAccount.getName() + ":")); } - throwIfCancelled(); - profile.deleteNotPresentAccounts(db); - throwIfCancelled(); - db.setTransactionSuccessful(); - Data.accounts.set(accountList); - } - finally { - db.endTransaction(); + prevAccount = acc; } + throwIfCancelled(); + + profile.deleteNotPresentAccounts(db); + throwIfCancelled(); + db.setTransactionSuccessful(); + listFilledOK = true; + } + finally { + db.endTransaction(); } } + // should not be set in the DB transaction, because of a possible deadlock + // with the main and DbOpQueueRunner threads + if (listFilledOK) Data.accounts.setList(accountList); return true; } @@ -429,68 +499,100 @@ public class RetrieveTransactionsTask default: throw new HTTPException(http.getResponseCode(), http.getResponseMessage()); } - try (SQLiteDatabase db = MLDB.getWritableDatabase()) { - try (InputStream resp = http.getInputStream()) { - if (http.getResponseCode() != 200) - throw new IOException(String.format("HTTP error %d", http.getResponseCode())); - throwIfCancelled(); - db.beginTransaction(); - try { - profile.markTransactionsAsNotPresent(db); + SQLiteDatabase db = MLDB.getDatabase(); + try (InputStream resp = http.getInputStream()) { + if (http.getResponseCode() != 200) + throw new IOException(String.format("HTTP error %d", http.getResponseCode())); + throwIfCancelled(); + db.beginTransaction(); + try { + profile.markTransactionsAsNotPresent(db); - int matchedTransactionsCount = 0; - TransactionListParser parser = new TransactionListParser(resp); + int matchedTransactionsCount = 0; + TransactionListParser parser = new TransactionListParser(resp); - int processedTransactionCount = 0; + int processedTransactionCount = 0; - while (true) { - throwIfCancelled(); - ParsedLedgerTransaction parsedTransaction = parser.nextTransaction(); - throwIfCancelled(); - if (parsedTransaction == null) break; - LedgerTransaction transaction = parsedTransaction.asLedgerTransaction(); - if (transaction.existsInDb(db)) { - profile.markTransactionAsPresent(db, transaction); - matchedTransactionsCount++; - - if (matchedTransactionsCount == MATCHING_TRANSACTIONS_LIMIT) { - profile.markTransactionsBeforeTransactionAsPresent(db, transaction); - progress.setTotal(progress.getProgress()); - publishProgress(progress); - db.setTransactionSuccessful(); - profile.setLastUpdateStamp(); - return true; - } + DetectedTransactionOrder transactionOrder = DetectedTransactionOrder.UNKNOWN; + int orderAccumulator = 0; + int lastTransactionId = 0; + + while (true) { + throwIfCancelled(); + ParsedLedgerTransaction parsedTransaction = parser.nextTransaction(); + throwIfCancelled(); + if (parsedTransaction == null) break; + + LedgerTransaction transaction = parsedTransaction.asLedgerTransaction(); + if (transaction.getId() > lastTransactionId) orderAccumulator++; + else orderAccumulator--; + lastTransactionId = transaction.getId(); + if (transactionOrder == DetectedTransactionOrder.UNKNOWN) { + if (orderAccumulator > 30) { + transactionOrder = DetectedTransactionOrder.FILE; + debug("rtt", String.format(Locale.ENGLISH, + "Detected native file order after %d transactions (factor %d)", + processedTransactionCount, orderAccumulator)); + progress.setTotal(Data.transactions.size()); } - else { - profile.storeTransaction(db, transaction); - matchedTransactionsCount = 0; - progress.setTotal(maxTransactionId); + else if (orderAccumulator < -30) { + transactionOrder = DetectedTransactionOrder.REVERSE_CHRONOLOGICAL; + debug("rtt", String.format(Locale.ENGLISH, + "Detected reverse chronological order after %d transactions (factor %d)", + processedTransactionCount, orderAccumulator)); } + } - progress.setProgress(++processedTransactionCount); - publishProgress(progress); + if (transaction.existsInDb(db)) { + profile.markTransactionAsPresent(db, transaction); + matchedTransactionsCount++; + + if ((transactionOrder == DetectedTransactionOrder.REVERSE_CHRONOLOGICAL) && + (matchedTransactionsCount == MATCHING_TRANSACTIONS_LIMIT)) + { + profile.markTransactionsBeforeTransactionAsPresent(db, transaction); + progress.setTotal(progress.getProgress()); + publishProgress(progress); + db.setTransactionSuccessful(); + profile.setLastUpdateStamp(); + return true; + } + } + else { + profile.storeTransaction(db, transaction); + matchedTransactionsCount = 0; + progress.setTotal(maxTransactionId); } - throwIfCancelled(); - profile.deleteNotPresentTransactions(db); - throwIfCancelled(); - db.setTransactionSuccessful(); - profile.setLastUpdateStamp(); - } - finally { - db.endTransaction(); + + if ((transactionOrder != DetectedTransactionOrder.UNKNOWN) && + ((progress.getTotal() == Progress.INDETERMINATE) || + (progress.getTotal() < transaction.getId()))) + progress.setTotal(transaction.getId()); + + progress.setProgress(++processedTransactionCount); + publishProgress(progress); } + + throwIfCancelled(); + profile.deleteNotPresentTransactions(db); + throwIfCancelled(); + db.setTransactionSuccessful(); + profile.setLastUpdateStamp(); + } + finally { + db.endTransaction(); } } return true; } + @SuppressLint("DefaultLocale") @Override protected String doInBackground(Void... params) { MobileLedgerProfile profile = Data.profile.get(); - Data.backgroundTaskCount.incrementAndGet(); + Data.backgroundTaskStarted(); try { if (!retrieveAccountList(profile) || !retrieveTransactionList(profile)) return retrieveTransactionListLegacy(profile); @@ -506,7 +608,7 @@ public class RetrieveTransactionsTask } catch (IOException e) { e.printStackTrace(); - return "Parse error"; + return e.getLocalizedMessage(); } catch (ParseException e) { e.printStackTrace(); @@ -517,7 +619,7 @@ public class RetrieveTransactionsTask return "Operation cancelled"; } finally { - Data.backgroundTaskCount.decrementAndGet(); + Data.backgroundTaskFinished(); } } private MainActivity getContext() { @@ -526,6 +628,7 @@ public class RetrieveTransactionsTask private void throwIfCancelled() { if (isCancelled()) throw new OperationCanceledException(null); } + enum DetectedTransactionOrder {UNKNOWN, REVERSE_CHRONOLOGICAL, FILE} private enum ParserState { EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_JOURNAL, EXPECTING_TRANSACTION,