app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java

   1 /*
   2  * Copyright © 2019 Damyan Ivanov.
   3  * This file is part of MoLe.
   4  * MoLe is free software: you can distribute it and/or modify it
   5  * under the term of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your opinion), any later version.
   8  *
   9  * MoLe is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12  * GNU General Public License terms for details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
  16  */
  17
  18 package net.ktnx.mobileledger.async;
  19
  20 import android.annotation.SuppressLint;
  21 import android.database.sqlite.SQLiteDatabase;
  22 import android.os.AsyncTask;
  23 import android.os.OperationCanceledException;
  24 import android.util.Log;
  25
  26 import net.ktnx.mobileledger.err.HTTPException;
  27 import net.ktnx.mobileledger.json.AccountListParser;
  28 import net.ktnx.mobileledger.json.ParsedBalance;
  29 import net.ktnx.mobileledger.json.ParsedLedgerAccount;
  30 import net.ktnx.mobileledger.json.ParsedLedgerTransaction;
  31 import net.ktnx.mobileledger.json.TransactionListParser;
  32 import net.ktnx.mobileledger.model.Data;
  33 import net.ktnx.mobileledger.model.LedgerAccount;
  34 import net.ktnx.mobileledger.model.LedgerTransaction;
  35 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
  36 import net.ktnx.mobileledger.model.MobileLedgerProfile;
  37 import net.ktnx.mobileledger.ui.activity.MainActivity;
  38 import net.ktnx.mobileledger.utils.MLDB;
  39 import net.ktnx.mobileledger.utils.NetworkUtil;
  40
  41 import java.io.BufferedReader;
  42 import java.io.IOException;
  43 import java.io.InputStream;
  44 import java.io.InputStreamReader;
  45 import java.lang.ref.WeakReference;
  46 import java.net.HttpURLConnection;
  47 import java.net.MalformedURLException;
  48 import java.net.URLDecoder;
  49 import java.nio.charset.StandardCharsets;
  50 import java.text.ParseException;
  51 import java.util.ArrayList;
  52 import java.util.HashMap;
  53 import java.util.Stack;
  54 import java.util.regex.Matcher;
  55 import java.util.regex.Pattern;
  56
  57
  58 public class RetrieveTransactionsTask
  59         extends AsyncTask<Void, RetrieveTransactionsTask.Progress, String> {
  60     private static final int MATCHING_TRANSACTIONS_LIMIT = 50;
  61     private static final Pattern reComment = Pattern.compile("^\\s*;");
  62     private static final Pattern reTransactionStart = Pattern.compile("<tr class=\"title\" " +
  63                                                                       "id=\"transaction-(\\d+)\"><td class=\"date\"[^\"]*>([\\d.-]+)</td>");
  64     private static final Pattern reTransactionDescription =
  65             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
  66     private static final Pattern reTransactionDetails =
  67             Pattern.compile("^\\s+(\\S[\\S\\s]+\\S)\\s\\s+([-+]?\\d[\\d,.]*)(?:\\s+(\\S+)$)?");
  68     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
  69     private WeakReference<MainActivity> contextRef;
  70     private int error;
  71     // %3A is '='
  72     private Pattern reAccountName = Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
  73     private Pattern reAccountValue = Pattern.compile(
  74             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
  75     public RetrieveTransactionsTask(WeakReference<MainActivity> contextRef) {
  76         this.contextRef = contextRef;
  77     }
  78     private static void L(String msg) {
  79         //Log.d("transaction-parser", msg);
  80     }
  81     @Override
  82     protected void onProgressUpdate(Progress... values) {
  83         super.onProgressUpdate(values);
  84         MainActivity context = getContext();
  85         if (context == null) return;
  86         context.onRetrieveProgress(values[0]);
  87     }
  88     @Override
  89     protected void onPreExecute() {
  90         super.onPreExecute();
  91         MainActivity context = getContext();
  92         if (context == null) return;
  93         context.onRetrieveStart();
  94     }
  95     @Override
  96     protected void onPostExecute(String error) {
  97         super.onPostExecute(error);
  98         MainActivity context = getContext();
  99         if (context == null) return;
 100         context.onRetrieveDone(error);
 101     }
 102     @Override
 103     protected void onCancelled() {
 104         super.onCancelled();
 105         MainActivity context = getContext();
 106         if (context == null) return;
 107         context.onRetrieveDone(null);
 108     }
 109     private String retrieveTransactionListLegacy(MobileLedgerProfile profile)
 110             throws IOException, ParseException, HTTPException {
 111         Progress progress = new Progress();
 112         int maxTransactionId = Progress.INDETERMINATE;
 113         ArrayList<LedgerAccount> accountList = new ArrayList<>();
 114         HashMap<String, Void> accountNames = new HashMap<>();
 115         LedgerAccount lastAccount = null, prevAccount = null;
 116         boolean onlyStarred = Data.optShowOnlyStarred.get();
 117
 118         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
 119         http.setAllowUserInteraction(false);
 120         publishProgress(progress);
 121         switch (http.getResponseCode()) {
 122             case 200:
 123                 break;
 124             default:
 125                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 126         }
 127         try (SQLiteDatabase db = MLDB.getWritableDatabase()) {
 128             try (InputStream resp = http.getInputStream()) {
 129                 if (http.getResponseCode() != 200)
 130                     throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 131                 db.beginTransaction();
 132                 try {
 133                     prepareDbForRetrieval(db, profile);
 134
 135                     int matchedTransactionsCount = 0;
 136
 137
 138                     ParserState state = ParserState.EXPECTING_ACCOUNT;
 139                     String line;
 140                     BufferedReader buf =
 141                             new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
 142
 143                     int processedTransactionCount = 0;
 144                     int transactionId = 0;
 145                     LedgerTransaction transaction = null;
 146                     LINES:
 147                     while ((line = buf.readLine()) != null) {
 148                         throwIfCancelled();
 149                         Matcher m;
 150                         m = reComment.matcher(line);
 151                         if (m.find()) {
 152                             // TODO: comments are ignored for now
 153                             Log.v("transaction-parser", "Ignoring comment");
 154                             continue;
 155                         }
 156                         //L(String.format("State is %d", updating));
 157                         switch (state) {
 158                             case EXPECTING_ACCOUNT:
 159                                 if (line.equals("<h2>General Journal</h2>")) {
 160                                     state = ParserState.EXPECTING_TRANSACTION;
 161                                     L("→ expecting transaction");
 162                                     Data.accounts.set(accountList);
 163                                     continue;
 164                                 }
 165                                 m = reAccountName.matcher(line);
 166                                 if (m.find()) {
 167                                     String acct_encoded = m.group(1);
 168                                     String acct_name = URLDecoder.decode(acct_encoded, "UTF-8");
 169                                     acct_name = acct_name.replace("\"", "");
 170                                     L(String.format("found account: %s", acct_name));
 171
 172                                     prevAccount = lastAccount;
 173                                     lastAccount = profile.tryLoadAccount(db, acct_name);
 174                                     if (lastAccount == null)
 175                                         lastAccount = new LedgerAccount(acct_name);
 176                                     else lastAccount.removeAmounts();
 177                                     profile.storeAccount(db, lastAccount);
 178
 179                                     if (prevAccount != null) prevAccount
 180                                             .setHasSubAccounts(prevAccount.isParentOf(lastAccount));
 181                                     // make sure the parent account(s) are present,
 182                                     // synthesising them if necessary
 183                                     String parentName = lastAccount.getParentName();
 184                                     if (parentName != null) {
 185                                         Stack<String> toAppend = new Stack<>();
 186                                         while (parentName != null) {
 187                                             if (accountNames.containsKey(parentName)) break;
 188                                             toAppend.push(parentName);
 189                                             parentName =
 190                                                     new LedgerAccount(parentName).getParentName();
 191                                         }
 192                                         while (!toAppend.isEmpty()) {
 193                                             String aName = toAppend.pop();
 194                                             LedgerAccount acc = new LedgerAccount(aName);
 195                                             acc.setHiddenByStar(lastAccount.isHiddenByStar());
 196                                             acc.setHasSubAccounts(true);
 197                                             if ((!onlyStarred || !acc.isHiddenByStar()) &&
 198                                                 acc.isVisible(accountList)) accountList.add(acc);
 199                                             L(String.format("gap-filling with %s", aName));
 200                                             accountNames.put(aName, null);
 201                                             profile.storeAccount(db, acc);
 202                                         }
 203                                     }
 204
 205                                     if ((!onlyStarred || !lastAccount.isHiddenByStar()) &&
 206                                         lastAccount.isVisible(accountList))
 207                                         accountList.add(lastAccount);
 208                                     accountNames.put(acct_name, null);
 209
 210                                     state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
 211                                     L("→ expecting account amount");
 212                                 }
 213                                 break;
 214
 215                             case EXPECTING_ACCOUNT_AMOUNT:
 216                                 m = reAccountValue.matcher(line);
 217                                 boolean match_found = false;
 218                                 while (m.find()) {
 219                                     throwIfCancelled();
 220
 221                                     match_found = true;
 222                                     String value = m.group(1);
 223                                     String currency = m.group(2);
 224                                     if (currency == null) currency = "";
 225                                     value = value.replace(',', '.');
 226                                     L("curr=" + currency + ", value=" + value);
 227                                     profile.storeAccountValue(db, lastAccount.getName(), currency,
 228                                             Float.valueOf(value));
 229                                     lastAccount.addAmount(Float.parseFloat(value), currency);
 230                                 }
 231
 232                                 if (match_found) {
 233                                     state = ParserState.EXPECTING_ACCOUNT;
 234                                     L("→ expecting account");
 235                                 }
 236
 237                                 break;
 238
 239                             case EXPECTING_TRANSACTION:
 240                                 if (!line.isEmpty() && (line.charAt(0) == ' ')) continue;
 241                                 m = reTransactionStart.matcher(line);
 242                                 if (m.find()) {
 243                                     transactionId = Integer.valueOf(m.group(1));
 244                                     state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
 245                                     L(String.format("found transaction %d → expecting description",
 246                                             transactionId));
 247                                     progress.setProgress(++processedTransactionCount);
 248                                     if (maxTransactionId < transactionId)
 249                                         maxTransactionId = transactionId;
 250                                     if ((progress.getTotal() == Progress.INDETERMINATE) ||
 251                                         (progress.getTotal() < transactionId))
 252                                         progress.setTotal(transactionId);
 253                                     publishProgress(progress);
 254                                 }
 255                                 m = reEnd.matcher(line);
 256                                 if (m.find()) {
 257                                     L("--- transaction value complete ---");
 258                                     break LINES;
 259                                 }
 260                                 break;
 261
 262                             case EXPECTING_TRANSACTION_DESCRIPTION:
 263                                 if (!line.isEmpty() && (line.charAt(0) == ' ')) continue;
 264                                 m = reTransactionDescription.matcher(line);
 265                                 if (m.find()) {
 266                                     if (transactionId == 0) throw new TransactionParserException(
 267                                             "Transaction Id is 0 while expecting " + "description");
 268
 269                                     String date = m.group(1);
 270                                     try {
 271                                         int equalsIndex = date.indexOf('=');
 272                                         if (equalsIndex >= 0)
 273                                             date = date.substring(equalsIndex + 1);
 274                                         transaction = new LedgerTransaction(transactionId, date,
 275                                                 m.group(2));
 276                                     }
 277                                     catch (ParseException e) {
 278                                         e.printStackTrace();
 279                                         return String.format("Error parsing date '%s'", date);
 280                                     }
 281                                     state = ParserState.EXPECTING_TRANSACTION_DETAILS;
 282                                     L(String.format("transaction %d created for %s (%s) →" +
 283                                                     " expecting details", transactionId, date,
 284                                             m.group(2)));
 285                                 }
 286                                 break;
 287
 288                             case EXPECTING_TRANSACTION_DETAILS:
 289                                 if (line.isEmpty()) {
 290                                     // transaction data collected
 291                                     if (transaction.existsInDb(db)) {
 292                                         profile.markTransactionAsPresent(db, transaction);
 293                                         matchedTransactionsCount++;
 294
 295                                         if (matchedTransactionsCount ==
 296                                             MATCHING_TRANSACTIONS_LIMIT)
 297                                         {
 298                                             profile.markTransactionsBeforeTransactionAsPresent(db,
 299                                                     transaction);
 300                                             progress.setTotal(progress.getProgress());
 301                                             publishProgress(progress);
 302                                             break LINES;
 303                                         }
 304                                     }
 305                                     else {
 306                                         profile.storeTransaction(db, transaction);
 307                                         matchedTransactionsCount = 0;
 308                                         progress.setTotal(maxTransactionId);
 309                                     }
 310
 311                                     state = ParserState.EXPECTING_TRANSACTION;
 312                                     L(String.format("transaction %s saved → expecting transaction",
 313                                             transaction.getId()));
 314                                     transaction.finishLoading();
 315
 316 // sounds like a good idea, but transaction-1 may not be the first one chronologically
 317 // for example, when you add the initial seeding transaction after entering some others
 318 //                                            if (transactionId == 1) {
 319 //                                                L("This was the initial transaction. Terminating " +
 320 //                                                  "parser");
 321 //                                                break LINES;
 322 //                                            }
 323                                 }
 324                                 else {
 325                                     m = reTransactionDetails.matcher(line);
 326                                     if (m.find()) {
 327                                         String acc_name = m.group(1);
 328                                         String amount = m.group(2);
 329                                         String currency = m.group(3);
 330                                         if (currency == null) currency = "";
 331                                         amount = amount.replace(',', '.');
 332                                         transaction.addAccount(
 333                                                 new LedgerTransactionAccount(acc_name,
 334                                                         Float.valueOf(amount), currency));
 335                                         L(String.format("%d: %s = %s", transaction.getId(),
 336                                                 acc_name, amount));
 337                                     }
 338                                     else throw new IllegalStateException(String.format(
 339                                             "Can't parse transaction %d " + "details: %s",
 340                                             transactionId, line));
 341                                 }
 342                                 break;
 343                             default:
 344                                 throw new RuntimeException(
 345                                         String.format("Unknown parser updating %s", state.name()));
 346                         }
 347                     }
 348
 349                     throwIfCancelled();
 350
 351                     profile.deleteNotPresentTransactions(db);
 352                     db.setTransactionSuccessful();
 353
 354                     profile.setLastUpdateStamp();
 355
 356                     return null;
 357                 }
 358                 finally {
 359                     db.endTransaction();
 360                 }
 361             }
 362         }
 363     }
 364     private void prepareDbForRetrieval(SQLiteDatabase db, MobileLedgerProfile profile) {
 365         db.execSQL("UPDATE transactions set keep=0 where profile=?",
 366                 new String[]{profile.getUuid()});
 367         db.execSQL("update account_values set keep=0 where profile=?;",
 368                 new String[]{profile.getUuid()});
 369         db.execSQL("update accounts set keep=0 where profile=?;", new String[]{profile.getUuid()});
 370     }
 371     private boolean retrieveAccountList(MobileLedgerProfile profile)
 372             throws IOException, HTTPException {
 373         Progress progress = new Progress();
 374
 375         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
 376         http.setAllowUserInteraction(false);
 377         switch (http.getResponseCode()) {
 378             case 200:
 379                 break;
 380             case 404:
 381                 return false;
 382             default:
 383                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 384         }
 385         publishProgress(progress);
 386         try (SQLiteDatabase db = MLDB.getWritableDatabase()) {
 387             try (InputStream resp = http.getInputStream()) {
 388                 if (http.getResponseCode() != 200)
 389                     throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 390
 391                 db.beginTransaction();
 392                 try {
 393                     profile.markAccountsAsNotPresent(db);
 394
 395                     AccountListParser parser = new AccountListParser(resp);
 396                     ArrayList<LedgerAccount> accountList = new ArrayList<>();
 397
 398                     LedgerAccount prevAccount = null;
 399
 400                     while (true) {
 401                         throwIfCancelled();
 402                         ParsedLedgerAccount parsedAccount = parser.nextAccount();
 403                         if (parsedAccount == null) break;
 404
 405                         LedgerAccount acc = profile.tryLoadAccount(db, parsedAccount.getAname());
 406                         if (acc == null) acc = new LedgerAccount(parsedAccount.getAname());
 407                         else acc.removeAmounts();
 408
 409                         profile.storeAccount(db, acc);
 410                         for (ParsedBalance b : parsedAccount.getAebalance()) {
 411                             profile.storeAccountValue(db, acc.getName(), b.getAcommodity(),
 412                                     b.getAquantity().asFloat());
 413                         }
 414
 415                         if (acc.isVisible(accountList)) accountList.add(acc);
 416
 417                         if (prevAccount != null) {
 418                             prevAccount.setHasSubAccounts(
 419                                     acc.getName().startsWith(prevAccount.getName() + ":"));
 420                         }
 421
 422                         prevAccount = acc;
 423                     }
 424                     throwIfCancelled();
 425
 426                     profile.deleteNotPresentAccounts(db);
 427                     throwIfCancelled();
 428                     db.setTransactionSuccessful();
 429                     Data.accounts.set(accountList);
 430                 }
 431                 finally {
 432                     db.endTransaction();
 433                 }
 434             }
 435         }
 436
 437         return true;
 438     }
 439     private boolean retrieveTransactionList(MobileLedgerProfile profile)
 440             throws IOException, ParseException, HTTPException {
 441         Progress progress = new Progress();
 442         int maxTransactionId = Progress.INDETERMINATE;
 443
 444         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
 445         http.setAllowUserInteraction(false);
 446         publishProgress(progress);
 447         switch (http.getResponseCode()) {
 448             case 200:
 449                 break;
 450             case 404:
 451                 return false;
 452             default:
 453                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 454         }
 455         try (SQLiteDatabase db = MLDB.getWritableDatabase()) {
 456             try (InputStream resp = http.getInputStream()) {
 457                 if (http.getResponseCode() != 200)
 458                     throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 459                 throwIfCancelled();
 460                 db.beginTransaction();
 461                 try {
 462                     profile.markTransactionsAsNotPresent(db);
 463
 464                     int matchedTransactionsCount = 0;
 465                     TransactionListParser parser = new TransactionListParser(resp);
 466
 467                     int processedTransactionCount = 0;
 468
 469                     while (true) {
 470                         throwIfCancelled();
 471                         ParsedLedgerTransaction parsedTransaction = parser.nextTransaction();
 472                         throwIfCancelled();
 473                         if (parsedTransaction == null) break;
 474                         LedgerTransaction transaction = parsedTransaction.asLedgerTransaction();
 475                         if (transaction.existsInDb(db)) {
 476                             profile.markTransactionAsPresent(db, transaction);
 477                             matchedTransactionsCount++;
 478
 479                             if (matchedTransactionsCount == MATCHING_TRANSACTIONS_LIMIT) {
 480                                 profile.markTransactionsBeforeTransactionAsPresent(db, transaction);
 481                                 progress.setTotal(progress.getProgress());
 482                                 publishProgress(progress);
 483                                 db.setTransactionSuccessful();
 484                                 profile.setLastUpdateStamp();
 485                                 return true;
 486                             }
 487                         }
 488                         else {
 489                             profile.storeTransaction(db, transaction);
 490                             matchedTransactionsCount = 0;
 491                             progress.setTotal(maxTransactionId);
 492                         }
 493
 494                         if ((progress.getTotal() == Progress.INDETERMINATE) ||
 495                             (progress.getTotal() < transaction.getId()))
 496                             progress.setTotal(transaction.getId());
 497
 498                         progress.setProgress(++processedTransactionCount);
 499                         publishProgress(progress);
 500                     }
 501
 502                     throwIfCancelled();
 503                     profile.deleteNotPresentTransactions(db);
 504                     throwIfCancelled();
 505                     db.setTransactionSuccessful();
 506                     profile.setLastUpdateStamp();
 507                 }
 508                 finally {
 509                     db.endTransaction();
 510                 }
 511             }
 512         }
 513
 514         return true;
 515     }
 516     @SuppressLint("DefaultLocale")
 517     @Override
 518     protected String doInBackground(Void... params) {
 519         MobileLedgerProfile profile = Data.profile.get();
 520         Data.backgroundTaskCount.incrementAndGet();
 521         try {
 522             if (!retrieveAccountList(profile) || !retrieveTransactionList(profile))
 523                 return retrieveTransactionListLegacy(profile);
 524             return null;
 525         }
 526         catch (MalformedURLException e) {
 527             e.printStackTrace();
 528             return "Invalid server URL";
 529         }
 530         catch (HTTPException e) {
 531             e.printStackTrace();
 532             return String.format("HTTP error %d: %s", e.getResponseCode(), e.getResponseMessage());
 533         }
 534         catch (IOException e) {
 535             e.printStackTrace();
 536             return e.getLocalizedMessage();
 537         }
 538         catch (ParseException e) {
 539             e.printStackTrace();
 540             return "Network error";
 541         }
 542         catch (OperationCanceledException e) {
 543             e.printStackTrace();
 544             return "Operation cancelled";
 545         }
 546         finally {
 547             Data.backgroundTaskCount.decrementAndGet();
 548         }
 549     }
 550     private MainActivity getContext() {
 551         return contextRef.get();
 552     }
 553     private void throwIfCancelled() {
 554         if (isCancelled()) throw new OperationCanceledException(null);
 555     }
 556
 557     private enum ParserState {
 558         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_JOURNAL, EXPECTING_TRANSACTION,
 559         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
 560     }
 561
 562     public class Progress {
 563         public static final int INDETERMINATE = -1;
 564         private int progress;
 565         private int total;
 566         Progress() {
 567             this(INDETERMINATE, INDETERMINATE);
 568         }
 569         Progress(int progress, int total) {
 570             this.progress = progress;
 571             this.total = total;
 572         }
 573         public int getProgress() {
 574             return progress;
 575         }
 576         protected void setProgress(int progress) {
 577             this.progress = progress;
 578         }
 579         public int getTotal() {
 580             return total;
 581         }
 582         protected void setTotal(int total) {
 583             this.total = total;
 584         }
 585     }
 586
 587     private class TransactionParserException extends IllegalStateException {
 588         TransactionParserException(String message) {
 589             super(message);
 590         }
 591     }
 592 }