app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java

   1 /*
   2  * Copyright © 2019 Damyan Ivanov.
   3  * This file is part of Mobile-Ledger.
   4  * Mobile-Ledger is free software: you can distribute it and/or modify it
   5  * under the term of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your opinion), any later version.
   8  *
   9  * Mobile-Ledger is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12  * GNU General Public License terms for details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with Mobile-Ledger. If not, see <https://www.gnu.org/licenses/>.
  16  */
  17
  18 package net.ktnx.mobileledger.async;
  19
  20 import android.annotation.SuppressLint;
  21 import android.content.SharedPreferences;
  22 import android.database.sqlite.SQLiteDatabase;
  23 import android.os.AsyncTask;
  24 import android.os.OperationCanceledException;
  25 import android.util.Log;
  26
  27 import net.ktnx.mobileledger.R;
  28 import net.ktnx.mobileledger.model.Data;
  29 import net.ktnx.mobileledger.model.LedgerAccount;
  30 import net.ktnx.mobileledger.model.LedgerTransaction;
  31 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
  32 import net.ktnx.mobileledger.ui.activity.MainActivity;
  33 import net.ktnx.mobileledger.ui.transaction_list.TransactionListViewModel;
  34 import net.ktnx.mobileledger.utils.MLDB;
  35 import net.ktnx.mobileledger.utils.NetworkUtil;
  36
  37 import java.io.BufferedReader;
  38 import java.io.FileNotFoundException;
  39 import java.io.IOException;
  40 import java.io.InputStream;
  41 import java.io.InputStreamReader;
  42 import java.lang.ref.WeakReference;
  43 import java.net.HttpURLConnection;
  44 import java.net.MalformedURLException;
  45 import java.net.URLDecoder;
  46 import java.util.ArrayList;
  47 import java.util.Date;
  48 import java.util.List;
  49 import java.util.regex.Matcher;
  50 import java.util.regex.Pattern;
  51
  52
  53 public class RetrieveTransactionsTask extends
  54         AsyncTask<RetrieveTransactionsTask.Params, RetrieveTransactionsTask.Progress, Void> {
  55     private static final Pattern transactionStartPattern = Pattern.compile("<tr class=\"title\" " +
  56                                                                            "id=\"transaction-(\\d+)\"><td class=\"date\"[^\\\"]*>([\\d.-]+)</td>");
  57     private static final Pattern transactionDescriptionPattern =
  58             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
  59     private static final Pattern transactionDetailsPattern =
  60             Pattern.compile("^\\s+" + "(\\S[\\S\\s]+\\S)\\s\\s+([-+]?\\d[\\d,.]*)(?:\\s+(\\S+)$)?");
  61     private static final Pattern endPattern = Pattern.compile("\\bid=\"addmodal\"");
  62     protected WeakReference<MainActivity> contextRef;
  63     protected int error;
  64     // %3A is '='
  65     Pattern account_name_re = Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
  66     Pattern account_value_re = Pattern.compile(
  67             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
  68     Pattern tr_end_re = Pattern.compile("</tr>");
  69     Pattern descriptions_line_re = Pattern.compile("\\bdescriptionsSuggester\\s*=\\s*new\\b");
  70     Pattern description_items_re = Pattern.compile("\"value\":\"([^\"]+)\"");
  71     private boolean success;
  72     public RetrieveTransactionsTask(WeakReference<MainActivity> contextRef) {
  73         this.contextRef = contextRef;
  74     }
  75     private static final void L(String msg) {
  76         Log.d("transaction-parser", msg);
  77     }
  78     @Override
  79     protected void onProgressUpdate(Progress... values) {
  80         super.onProgressUpdate(values);
  81         MainActivity context = getContext();
  82         if (context == null) return;
  83         context.onRetrieveProgress(values[0]);
  84     }
  85     @Override
  86     protected void onPreExecute() {
  87         super.onPreExecute();
  88         MainActivity context = getContext();
  89         if (context == null) return;
  90         context.onRetrieveStart();
  91     }
  92     @Override
  93     protected void onPostExecute(Void aVoid) {
  94         super.onPostExecute(aVoid);
  95         MainActivity context = getContext();
  96         if (context == null) return;
  97         context.onRetrieveDone(success);
  98     }
  99     @Override
 100     protected void onCancelled() {
 101         super.onCancelled();
 102         MainActivity context = getContext();
 103         if (context == null) return;
 104         context.onRetrieveDone(false);
 105     }
 106     @SuppressLint("DefaultLocale")
 107     @Override
 108     protected Void doInBackground(Params... params) {
 109         Progress progress = new Progress();
 110         int maxTransactionId = Progress.INDETERMINATE;
 111         success = false;
 112         List<LedgerAccount> accountList = new ArrayList<>();
 113         LedgerAccount lastAccount = null;
 114         Data.backgroundTaskCount.incrementAndGet();
 115         try {
 116             HttpURLConnection http =
 117                     NetworkUtil.prepare_connection(params[0].getBackendPref(), "journal");
 118             http.setAllowUserInteraction(false);
 119             publishProgress(progress);
 120             MainActivity ctx = getContext();
 121             if (ctx == null) return null;
 122             try (SQLiteDatabase db = MLDB.getWritableDatabase()) {
 123                 try (InputStream resp = http.getInputStream()) {
 124                     if (http.getResponseCode() != 200) throw new IOException(
 125                             String.format("HTTP error %d", http.getResponseCode()));
 126                     db.beginTransaction();
 127                     try {
 128                         db.execSQL("UPDATE transactions set keep=0");
 129                         db.execSQL("update account_values set keep=0;");
 130                         db.execSQL("update accounts set keep=0;");
 131
 132                         ParserState state = ParserState.EXPECTING_ACCOUNT;
 133                         String line;
 134                         BufferedReader buf =
 135                                 new BufferedReader(new InputStreamReader(resp, "UTF-8"));
 136
 137                         int processedTransactionCount = 0;
 138                         int transactionId = 0;
 139                         int matchedTransactionsCount = 0;
 140                         LedgerTransaction transaction = null;
 141                         LINES:
 142                         while ((line = buf.readLine()) != null) {
 143                             throwIfCancelled();
 144                             Matcher m;
 145                             //L(String.format("State is %d", updating));
 146                             switch (state) {
 147                                 case EXPECTING_ACCOUNT:
 148                                     if (line.equals("<h2>General Journal</h2>")) {
 149                                         state = ParserState.EXPECTING_TRANSACTION;
 150                                         L("→ expecting transaction");
 151                                         Data.accounts.set(accountList);
 152                                         continue;
 153                                     }
 154                                     m = account_name_re.matcher(line);
 155                                     if (m.find()) {
 156                                         String acct_encoded = m.group(1);
 157                                         String acct_name = URLDecoder.decode(acct_encoded, "UTF-8");
 158                                         acct_name = acct_name.replace("\"", "");
 159                                         L(String.format("found account: %s", acct_name));
 160
 161                                         addAccount(db, acct_name);
 162                                         lastAccount = new LedgerAccount(acct_name);
 163                                         accountList.add(lastAccount);
 164
 165                                         state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
 166                                         L("→ expecting account amount");
 167                                     }
 168                                     break;
 169
 170                                 case EXPECTING_ACCOUNT_AMOUNT:
 171                                     m = account_value_re.matcher(line);
 172                                     boolean match_found = false;
 173                                     while (m.find()) {
 174                                         throwIfCancelled();
 175
 176                                         match_found = true;
 177                                         String value = m.group(1);
 178                                         String currency = m.group(2);
 179                                         if (currency == null) currency = "";
 180                                         value = value.replace(',', '.');
 181                                         L("curr=" + currency + ", value=" + value);
 182                                         db.execSQL(
 183                                                 "insert or replace into account_values(account, currency, value, keep) values(?, ?, ?, 1);",
 184                                                 new Object[]{lastAccount.getName(),
 185                                                              currency,
 186                                                              Float.valueOf(value)
 187                                                 });
 188                                         lastAccount.addAmount(Float.parseFloat(value), currency);
 189                                     }
 190
 191                                     if (match_found) {
 192                                         state = ParserState.EXPECTING_ACCOUNT;
 193                                         L("→ expecting account");
 194                                     }
 195
 196                                     break;
 197
 198                                 case EXPECTING_TRANSACTION:
 199                                     if (!line.isEmpty() && (line.charAt(0) == ' ')) continue;
 200                                     m = transactionStartPattern.matcher(line);
 201                                     if (m.find()) {
 202                                         transactionId = Integer.valueOf(m.group(1));
 203                                         state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
 204                                         L(String.format(
 205                                                 "found transaction %d → expecting description",
 206                                                 transactionId));
 207                                         progress.setProgress(++processedTransactionCount);
 208                                         if (maxTransactionId < transactionId)
 209                                             maxTransactionId = transactionId;
 210                                         if ((progress.getTotal() == Progress.INDETERMINATE) ||
 211                                             (progress.getTotal() < transactionId))
 212                                             progress.setTotal(transactionId);
 213                                         publishProgress(progress);
 214                                     }
 215                                     m = endPattern.matcher(line);
 216                                     if (m.find()) {
 217                                         L("--- transaction value complete ---");
 218                                         success = true;
 219                                         break LINES;
 220                                     }
 221                                     break;
 222
 223                                 case EXPECTING_TRANSACTION_DESCRIPTION:
 224                                     if (!line.isEmpty() && (line.charAt(0) == ' ')) continue;
 225                                     m = transactionDescriptionPattern.matcher(line);
 226                                     if (m.find()) {
 227                                         if (transactionId == 0)
 228                                             throw new TransactionParserException(
 229                                                     "Transaction Id is 0 while expecting " +
 230                                                     "description");
 231
 232                                         transaction =
 233                                                 new LedgerTransaction(transactionId, m.group(1),
 234                                                         m.group(2));
 235                                         state = ParserState.EXPECTING_TRANSACTION_DETAILS;
 236                                         L(String.format("transaction %d created for %s (%s) →" +
 237                                                         " expecting details", transactionId,
 238                                                 m.group(1), m.group(2)));
 239                                     }
 240                                     break;
 241
 242                                 case EXPECTING_TRANSACTION_DETAILS:
 243                                     if (line.isEmpty()) {
 244                                         // transaction data collected
 245                                         if (transaction.existsInDb(db)) {
 246                                             db.execSQL("UPDATE transactions SET keep = 1 WHERE id" +
 247                                                        "=?", new Integer[]{transaction.getId()});
 248                                             matchedTransactionsCount++;
 249
 250                                             if (matchedTransactionsCount == 100) {
 251                                                 db.execSQL("UPDATE transactions SET keep=1 WHERE " +
 252                                                            "id < ?",
 253                                                         new Integer[]{transaction.getId()});
 254                                                 success = true;
 255                                                 progress.setTotal(progress.getProgress());
 256                                                 publishProgress(progress);
 257                                                 break LINES;
 258                                             }
 259                                         }
 260                                         else {
 261                                             db.execSQL("DELETE from transactions WHERE id=?",
 262                                                     new Integer[]{transaction.getId()});
 263                                             db.execSQL("DELETE from transaction_accounts WHERE " +
 264                                                        "transaction_id=?",
 265                                                     new Integer[]{transaction.getId()});
 266                                             transaction.insertInto(db);
 267                                             matchedTransactionsCount = 0;
 268                                             progress.setTotal(maxTransactionId);
 269                                         }
 270
 271                                         state = ParserState.EXPECTING_TRANSACTION;
 272                                         L(String.format(
 273                                                 "transaction %s saved → expecting transaction",
 274                                                 transaction.getId()));
 275
 276 // sounds like a good idea, but transaction-1 may not be the first one chronologically
 277 // for example, when you add the initial seeding transaction after entering some others
 278 //                                            if (transactionId == 1) {
 279 //                                                L("This was the initial transaction. Terminating " +
 280 //                                                  "parser");
 281 //                                                break LINES;
 282 //                                            }
 283                                     }
 284                                     else {
 285                                         m = transactionDetailsPattern.matcher(line);
 286                                         if (m.find()) {
 287                                             String acc_name = m.group(1);
 288                                             String amount = m.group(2);
 289                                             String currency = m.group(3);
 290                                             amount = amount.replace(',', '.');
 291                                             transaction.addAccount(
 292                                                     new LedgerTransactionAccount(acc_name,
 293                                                             Float.valueOf(amount), currency));
 294                                             L(String.format("%s = %s", acc_name, amount));
 295                                         }
 296                                         else throw new IllegalStateException(
 297                                                 String.format("Can't parse transaction %d details",
 298                                                         transactionId));
 299                                     }
 300                                     break;
 301                                 default:
 302                                     throw new RuntimeException(
 303                                             String.format("Unknown parser updating %s", state.name()));
 304                             }
 305                         }
 306                         if (!isCancelled()) {
 307                             db.execSQL("DELETE FROM transactions WHERE keep = 0");
 308                             db.setTransactionSuccessful();
 309                         }
 310                     }
 311                     finally {
 312                         db.endTransaction();
 313                     }
 314                 }
 315             }
 316
 317             if (success && !isCancelled()) {
 318                 Log.d("db", "Updating transaction value stamp");
 319                 MLDB.set_option_value(MLDB.OPT_TRANSACTION_LIST_STAMP, new Date().getTime());
 320                 TransactionListViewModel.scheduleTransactionListReload(ctx);
 321             }
 322         }
 323         catch (MalformedURLException e) {
 324             error = R.string.err_bad_backend_url;
 325             e.printStackTrace();
 326         }
 327         catch (FileNotFoundException e) {
 328             error = R.string.err_bad_auth;
 329             e.printStackTrace();
 330         }
 331         catch (IOException e) {
 332             error = R.string.err_net_io_error;
 333             e.printStackTrace();
 334         }
 335         finally {
 336             Data.backgroundTaskCount.decrementAndGet();
 337         }
 338         return null;
 339     }
 340     private MainActivity getContext() {
 341         return contextRef.get();
 342     }
 343     private void addAccount(SQLiteDatabase db, String name) {
 344         do {
 345             LedgerAccount acc = new LedgerAccount(name);
 346             db.execSQL("update accounts set level = ?, keep = 1 where name = ?",
 347                     new Object[]{acc.getLevel(), name});
 348             db.execSQL("insert into accounts(name, name_upper, parent_name, level) select ?,?," +
 349                        "?,? " + "where (select changes() = 0)",
 350                     new Object[]{name, name.toUpperCase(), acc.getParentName(), acc.getLevel()});
 351             name = acc.getParentName();
 352         } while (name != null);
 353     }
 354     private void throwIfCancelled() {
 355         if (isCancelled()) throw new OperationCanceledException(null);
 356     }
 357
 358     private enum ParserState {
 359         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_JOURNAL, EXPECTING_TRANSACTION,
 360         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
 361     }
 362
 363     public static class Params {
 364         private SharedPreferences backendPref;
 365
 366         public Params(SharedPreferences backendPref) {
 367             this.backendPref = backendPref;
 368         }
 369         SharedPreferences getBackendPref() {
 370             return backendPref;
 371         }
 372     }
 373
 374     public class Progress {
 375         public static final int INDETERMINATE = -1;
 376         private int progress;
 377         private int total;
 378         Progress() {
 379             this(INDETERMINATE, INDETERMINATE);
 380         }
 381         Progress(int progress, int total) {
 382             this.progress = progress;
 383             this.total = total;
 384         }
 385         public int getProgress() {
 386             return progress;
 387         }
 388         protected void setProgress(int progress) {
 389             this.progress = progress;
 390         }
 391         public int getTotal() {
 392             return total;
 393         }
 394         protected void setTotal(int total) {
 395             this.total = total;
 396         }
 397     }
 398
 399     private class TransactionParserException extends IllegalStateException {
 400         TransactionParserException(String message) {
 401             super(message);
 402         }
 403     }
 404 }