X-Git-Url: https://git.ktnx.net/?p=mobile-ledger.git;a=blobdiff_plain;f=app%2Fsrc%2Fmain%2Fjava%2Fnet%2Fktnx%2Fmobileledger%2Fasync%2FRetrieveTransactionsTask.java;h=4da02e823111a070efa473edcbdf8c82ab8521b3;hp=aeb49a5f104b9ebcebda479b961fe83e855464db;hb=e0b2d33c159281741f3a6e30975d58040a8ab879;hpb=6b740c280c79b0170321f533747cdbfc3e179a29 diff --git a/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java b/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java index aeb49a5f..4da02e82 100644 --- a/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java +++ b/app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java @@ -1,5 +1,5 @@ /* - * Copyright © 2018 Damyan Ivanov. + * Copyright © 2019 Damyan Ivanov. * This file is part of Mobile-Ledger. * Mobile-Ledger is free software: you can distribute it and/or modify it * under the term of the GNU General Public License as published by @@ -17,15 +17,21 @@ package net.ktnx.mobileledger.async; -import android.content.Context; -import android.content.SharedPreferences; +import android.annotation.SuppressLint; import android.database.sqlite.SQLiteDatabase; import android.os.AsyncTask; +import android.os.OperationCanceledException; +import android.util.Log; import net.ktnx.mobileledger.R; +import net.ktnx.mobileledger.model.Data; +import net.ktnx.mobileledger.model.LedgerAccount; import net.ktnx.mobileledger.model.LedgerTransaction; -import net.ktnx.mobileledger.model.LedgerTransactionItem; -import net.ktnx.mobileledger.utils.MobileLedgerDatabase; +import net.ktnx.mobileledger.model.LedgerTransactionAccount; +import net.ktnx.mobileledger.model.MobileLedgerProfile; +import net.ktnx.mobileledger.ui.activity.MainActivity; +import net.ktnx.mobileledger.ui.transaction_list.TransactionListViewModel; +import net.ktnx.mobileledger.utils.MLDB; import net.ktnx.mobileledger.utils.NetworkUtil; import java.io.BufferedReader; @@ -36,149 +42,321 @@ import java.io.InputStreamReader; import java.lang.ref.WeakReference; import java.net.HttpURLConnection; import java.net.MalformedURLException; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; -class RetrieveTransactionsTask extends AsyncTask { - class Params { - static final int DEFAULT_LIMIT = 100; - private SharedPreferences backendPref; - private String accountsRoot; - private int limit; - - Params(SharedPreferences backendPref) { - this.backendPref = backendPref; - this.accountsRoot = null; - this.limit = DEFAULT_LIMIT; - } - Params(SharedPreferences backendPref, String accountsRoot) { - this(backendPref, accountsRoot, DEFAULT_LIMIT); - } - Params(SharedPreferences backendPref, String accountsRoot, int limit) { - this.backendPref = backendPref; - this.accountsRoot = accountsRoot; - this.limit = limit; - } - String getAccountsRoot() { - return accountsRoot; - } - SharedPreferences getBackendPref() { - return backendPref; - } - int getLimit() { - return limit; - } - } - private static final Pattern transactionStartPattern = Pattern.compile("([\\d.-]+)"); - private static final Pattern transactionDescriptionPattern = +public class RetrieveTransactionsTask + extends AsyncTask { + private static final int MATCHING_TRANSACTIONS_LIMIT = 50; + private static final Pattern reComment = Pattern.compile("^\\s*;"); + private static final Pattern reTransactionStart = Pattern.compile("([\\d.-]+)"); + private static final Pattern reTransactionDescription = Pattern.compile(" contextRef; - protected int error; + private static final Pattern reTransactionDetails = + Pattern.compile("^\\s+(\\S[\\S\\s]+\\S)\\s\\s+([-+]?\\d[\\d,.]*)(?:\\s+(\\S+)$)?"); + private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\""); + private WeakReference contextRef; + private int error; + // %3A is '=' + private boolean success; + private Pattern reAccountName = Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\""); + private Pattern reAccountValue = Pattern.compile( + "\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?"); + public RetrieveTransactionsTask(WeakReference contextRef) { + this.contextRef = contextRef; + } + private static void L(String msg) { + Log.d("transaction-parser", msg); + } + @Override + protected void onProgressUpdate(Progress... values) { + super.onProgressUpdate(values); + MainActivity context = getContext(); + if (context == null) return; + context.onRetrieveProgress(values[0]); + } + @Override + protected void onPreExecute() { + super.onPreExecute(); + MainActivity context = getContext(); + if (context == null) return; + context.onRetrieveStart(); + } + @Override + protected void onPostExecute(Void aVoid) { + super.onPostExecute(aVoid); + MainActivity context = getContext(); + if (context == null) return; + context.onRetrieveDone(success); + } + @Override + protected void onCancelled() { + super.onCancelled(); + MainActivity context = getContext(); + if (context == null) return; + context.onRetrieveDone(false); + } + @SuppressLint("DefaultLocale") @Override - protected Void doInBackground(Params... params) { + protected Void doInBackground(Void... params) { + MobileLedgerProfile profile = Data.profile.get(); + Progress progress = new Progress(); + int maxTransactionId = Progress.INDETERMINATE; + success = false; + ArrayList accountList = new ArrayList<>(); + HashMap accountNames = new HashMap<>(); + LedgerAccount lastAccount = null; + boolean onlyStarred = Data.optShowOnlyStarred.get(); + Data.backgroundTaskCount.incrementAndGet(); try { - HttpURLConnection http = - NetworkUtil.prepare_connection(params[0].getBackendPref(), "journal"); + HttpURLConnection http = NetworkUtil.prepareConnection("journal"); http.setAllowUserInteraction(false); - publishProgress(0); - Context ctx = contextRef.get(); + publishProgress(progress); + MainActivity ctx = getContext(); if (ctx == null) return null; - try (MobileLedgerDatabase dbh = new MobileLedgerDatabase(ctx)) { - try (SQLiteDatabase db = dbh.getWritableDatabase()) { - try (InputStream resp = http.getInputStream()) { - if (http.getResponseCode() != 200) throw new IOException( - String.format("HTTP error %d", http.getResponseCode())); - db.beginTransaction(); - try { - String root = params[0].getAccountsRoot(); - if (root == null) db.execSQL("DELETE FROM transaction_history;"); - else { - StringBuilder sql = new StringBuilder(); - sql.append("DELETE FROM transaction_history "); - sql.append( - "where id in (select transactions.id from transactions "); - sql.append("join transaction_accounts "); - sql.append( - "on transactions.id=transaction_accounts.transaction_id "); - sql.append("where transaction_accounts.account_name like ?||'%'"); - db.execSQL(sql.toString(), new String[]{root}); - } + try (SQLiteDatabase db = MLDB.getWritableDatabase()) { + try (InputStream resp = http.getInputStream()) { + if (http.getResponseCode() != 200) throw new IOException( + String.format("HTTP error %d", http.getResponseCode())); + db.beginTransaction(); + try { + db.execSQL("UPDATE transactions set keep=0"); + db.execSQL("update account_values set keep=0;"); + db.execSQL("update accounts set keep=0;"); + + ParserState state = ParserState.EXPECTING_ACCOUNT; + String line; + BufferedReader buf = + new BufferedReader(new InputStreamReader(resp, "UTF-8")); - int state = ParserState.EXPECTING_JOURNAL; - String line; - BufferedReader buf = - new BufferedReader(new InputStreamReader(resp, "UTF-8")); - - int transactionCount = 0; - String transactionId = null; - LedgerTransaction transaction = null; - while ((line = buf.readLine()) != null) { - switch (state) { - case ParserState.EXPECTING_JOURNAL: { - if (line.equals("

General Journal

")) - state = ParserState.EXPECTING_TRANSACTION; + int processedTransactionCount = 0; + int transactionId = 0; + int matchedTransactionsCount = 0; + LedgerTransaction transaction = null; + LINES: + while ((line = buf.readLine()) != null) { + throwIfCancelled(); + Matcher m; + m = reComment.matcher(line); + if (m.find()) { + // TODO: comments are ignored for now + Log.v("transaction-parser", "Ignoring comment"); + continue; + } + //L(String.format("State is %d", updating)); + switch (state) { + case EXPECTING_ACCOUNT: + if (line.equals("

General Journal

")) { + state = ParserState.EXPECTING_TRANSACTION; + L("→ expecting transaction"); + Data.accounts.set(accountList); continue; } - case ParserState.EXPECTING_TRANSACTION: { - Matcher m = transactionStartPattern.matcher(line); - if (m.find()) { - transactionId = m.group(1); - state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION; + m = reAccountName.matcher(line); + if (m.find()) { + String acct_encoded = m.group(1); + String acct_name = URLDecoder.decode(acct_encoded, "UTF-8"); + acct_name = acct_name.replace("\"", ""); + L(String.format("found account: %s", acct_name)); + + lastAccount = profile.loadAccount(acct_name); + if (lastAccount == null) { + lastAccount = new LedgerAccount(acct_name); + profile.storeAccount(lastAccount); } - } - case ParserState.EXPECTING_TRANSACTION_DESCRIPTION: { - Matcher m = transactionDescriptionPattern.matcher(line); - if (m.find()) { - if (transactionId == null) - throw new TransactionParserException( - "Transaction Id is null while expecting description"); - - transaction = - new LedgerTransaction(transactionId, m.group(1), - m.group(2)); - state = ParserState.EXPECTING_TRANSACTION_DETAILS; + + // make sure the parent account(s) are present, + // synthesising them if necessary + String parentName = lastAccount.getParentName(); + if (parentName != null) { + Stack toAppend = new Stack<>(); + while (parentName != null) { + if (accountNames.containsKey(parentName)) break; + toAppend.push(parentName); + parentName = new LedgerAccount(parentName) + .getParentName(); + } + while (!toAppend.isEmpty()) { + String aName = toAppend.pop(); + LedgerAccount acc = new LedgerAccount(aName); + acc.setHidden(lastAccount.isHidden()); + if (!onlyStarred || !acc.isHidden()) + accountList.add(acc); + L(String.format("gap-filling with %s", aName)); + accountNames.put(aName, null); + profile.storeAccount(acc); + } } + + if (!onlyStarred || !lastAccount.isHidden()) + accountList.add(lastAccount); + accountNames.put(acct_name, null); + + state = ParserState.EXPECTING_ACCOUNT_AMOUNT; + L("→ expecting account amount"); } - case ParserState.EXPECTING_TRANSACTION_DETAILS: { - if (transaction == null) + break; + + case EXPECTING_ACCOUNT_AMOUNT: + m = reAccountValue.matcher(line); + boolean match_found = false; + while (m.find()) { + throwIfCancelled(); + + match_found = true; + String value = m.group(1); + String currency = m.group(2); + if (currency == null) currency = ""; + value = value.replace(',', '.'); + L("curr=" + currency + ", value=" + value); + profile.storeAccountValue(lastAccount.getName(), currency, + Float.valueOf(value)); + lastAccount.addAmount(Float.parseFloat(value), currency); + } + + if (match_found) { + state = ParserState.EXPECTING_ACCOUNT; + L("→ expecting account"); + } + + break; + + case EXPECTING_TRANSACTION: + if (!line.isEmpty() && (line.charAt(0) == ' ')) continue; + m = reTransactionStart.matcher(line); + if (m.find()) { + transactionId = Integer.valueOf(m.group(1)); + state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION; + L(String.format( + "found transaction %d → expecting description", + transactionId)); + progress.setProgress(++processedTransactionCount); + if (maxTransactionId < transactionId) + maxTransactionId = transactionId; + if ((progress.getTotal() == Progress.INDETERMINATE) || + (progress.getTotal() < transactionId)) + progress.setTotal(transactionId); + publishProgress(progress); + } + m = reEnd.matcher(line); + if (m.find()) { + L("--- transaction value complete ---"); + success = true; + break LINES; + } + break; + + case EXPECTING_TRANSACTION_DESCRIPTION: + if (!line.isEmpty() && (line.charAt(0) == ' ')) continue; + m = reTransactionDescription.matcher(line); + if (m.find()) { + if (transactionId == 0) throw new TransactionParserException( - "Transaction is null while expecting details"); - if (line.isEmpty()) { - // transaction data collected - transaction.insertInto(db); + "Transaction Id is 0 while expecting " + + "description"); - state = ParserState.EXPECTING_TRANSACTION; - publishProgress(++transactionCount); + transaction = + new LedgerTransaction(transactionId, m.group(1), + m.group(2)); + state = ParserState.EXPECTING_TRANSACTION_DETAILS; + L(String.format("transaction %d created for %s (%s) →" + + " expecting details", transactionId, + m.group(1), m.group(2))); + } + break; + + case EXPECTING_TRANSACTION_DETAILS: + if (line.isEmpty()) { + // transaction data collected + if (transaction.existsInDb(db)) { + db.execSQL("UPDATE transactions SET keep = 1 WHERE " + + "profile = ? and id=?", + new Object[]{profile.getUuid(), + transaction.getId() + }); + matchedTransactionsCount++; + + if (matchedTransactionsCount == + MATCHING_TRANSACTIONS_LIMIT) + { + db.execSQL("UPDATE transactions SET keep=1 WHERE " + + "profile = ? and id < ?", + new Object[]{profile.getUuid(), + transaction.getId() + }); + success = true; + progress.setTotal(progress.getProgress()); + publishProgress(progress); + break LINES; + } } else { - Matcher m = transactionDetailsPattern.matcher(line); - if (m.find()) { - String acc_name = m.group(1); - String amount = m.group(2); - amount = amount.replace(',', '.'); - transaction.add_item( - new LedgerTransactionItem(acc_name, - Float.valueOf(amount))); - } - else throw new IllegalStateException(String.format( - "Can't" + " parse transaction details")); + profile.storeTransaction(transaction); + matchedTransactionsCount = 0; + progress.setTotal(maxTransactionId); } + + state = ParserState.EXPECTING_TRANSACTION; + L(String.format( + "transaction %s saved → expecting transaction", + transaction.getId())); + transaction.finishLoading(); + +// sounds like a good idea, but transaction-1 may not be the first one chronologically +// for example, when you add the initial seeding transaction after entering some others +// if (transactionId == 1) { +// L("This was the initial transaction. Terminating " + +// "parser"); +// break LINES; +// } + } + else { + m = reTransactionDetails.matcher(line); + if (m.find()) { + String acc_name = m.group(1); + String amount = m.group(2); + String currency = m.group(3); + if (currency == null) currency = ""; + amount = amount.replace(',', '.'); + transaction.addAccount( + new LedgerTransactionAccount(acc_name, + Float.valueOf(amount), currency)); + L(String.format("%d: %s = %s", transaction.getId(), + acc_name, amount)); + } + else throw new IllegalStateException(String.format( + "Can't parse transaction %d " + "details: %s", + transactionId, line)); } - default: - throw new RuntimeException( - String.format("Unknown " + "parser state %d", - state)); - } + break; + default: + throw new RuntimeException( + String.format("Unknown parser updating %s", + state.name())); } - db.setTransactionSuccessful(); - } - finally { - db.endTransaction(); } + + throwIfCancelled(); + + db.execSQL("DELETE FROM transactions WHERE profile=? AND keep = 0", + new String[]{profile.getUuid()}); + db.setTransactionSuccessful(); + + Log.d("db", "Updating transaction value stamp"); + Date now = new Date(); + profile.setLongOption(MLDB.OPT_LAST_SCRAPE, now.getTime()); + Data.lastUpdateDate.set(now); + TransactionListViewModel.scheduleTransactionListReload(); + } + finally { + db.endTransaction(); } } } @@ -195,10 +373,50 @@ class RetrieveTransactionsTask extends AsyncTask getContextRef() { - return contextRef; + private MainActivity getContext() { + return contextRef.get(); + } + private void throwIfCancelled() { + if (isCancelled()) throw new OperationCanceledException(null); + } + + private enum ParserState { + EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_JOURNAL, EXPECTING_TRANSACTION, + EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS + } + + public class Progress { + public static final int INDETERMINATE = -1; + private int progress; + private int total; + Progress() { + this(INDETERMINATE, INDETERMINATE); + } + Progress(int progress, int total) { + this.progress = progress; + this.total = total; + } + public int getProgress() { + return progress; + } + protected void setProgress(int progress) { + this.progress = progress; + } + public int getTotal() { + return total; + } + protected void setTotal(int total) { + this.total = total; + } } private class TransactionParserException extends IllegalStateException { @@ -206,11 +424,4 @@ class RetrieveTransactionsTask extends AsyncTask