app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java

   1 /*
   2  * Copyright © 2020 Damyan Ivanov.
   3  * This file is part of MoLe.
   4  * MoLe is free software: you can distribute it and/or modify it
   5  * under the term of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your opinion), any later version.
   8  *
   9  * MoLe is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12  * GNU General Public License terms for details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
  16  */
  17
  18 package net.ktnx.mobileledger.async;
  19
  20 import android.annotation.SuppressLint;
  21 import android.database.sqlite.SQLiteDatabase;
  22 import android.os.AsyncTask;
  23 import android.os.OperationCanceledException;
  24
  25 import androidx.annotation.NonNull;
  26
  27 import net.ktnx.mobileledger.App;
  28 import net.ktnx.mobileledger.err.HTTPException;
  29 import net.ktnx.mobileledger.json.v1_15.AccountListParser;
  30 import net.ktnx.mobileledger.json.v1_15.ParsedBalance;
  31 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerAccount;
  32 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerTransaction;
  33 import net.ktnx.mobileledger.json.v1_15.TransactionListParser;
  34 import net.ktnx.mobileledger.model.Data;
  35 import net.ktnx.mobileledger.model.LedgerAccount;
  36 import net.ktnx.mobileledger.model.LedgerTransaction;
  37 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
  38 import net.ktnx.mobileledger.model.MobileLedgerProfile;
  39 import net.ktnx.mobileledger.utils.NetworkUtil;
  40
  41 import java.io.BufferedReader;
  42 import java.io.IOException;
  43 import java.io.InputStream;
  44 import java.io.InputStreamReader;
  45 import java.net.HttpURLConnection;
  46 import java.net.MalformedURLException;
  47 import java.net.URLDecoder;
  48 import java.nio.charset.StandardCharsets;
  49 import java.text.ParseException;
  50 import java.util.ArrayList;
  51 import java.util.HashMap;
  52 import java.util.Locale;
  53 import java.util.Objects;
  54 import java.util.regex.Matcher;
  55 import java.util.regex.Pattern;
  56
  57
  58 public class RetrieveTransactionsTask
  59         extends AsyncTask<Void, RetrieveTransactionsTask.Progress, String> {
  60     private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
  61     private static final Pattern reComment = Pattern.compile("^\\s*;");
  62     private static final Pattern reTransactionStart = Pattern.compile(
  63             "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
  64             "\"[^\"]*>([\\d.-]+)</td>");
  65     private static final Pattern reTransactionDescription =
  66             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
  67     private static final Pattern reTransactionDetails = Pattern.compile(
  68             "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
  69             "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
  70     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
  71     private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
  72     private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
  73     // %3A is '='
  74     private Pattern reAccountName = Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
  75     private Pattern reAccountValue = Pattern.compile(
  76             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
  77     private MobileLedgerProfile profile;
  78     public RetrieveTransactionsTask(@NonNull MobileLedgerProfile profile) {
  79         this.profile = profile;
  80     }
  81     private static void L(String msg) {
  82         //debug("transaction-parser", msg);
  83     }
  84     static LedgerTransactionAccount parseTransactionAccountLine(String line) {
  85         Matcher m = reTransactionDetails.matcher(line);
  86         if (m.find()) {
  87             String postingStatus = m.group(1);
  88             String acc_name = m.group(2);
  89             String currencyPre = m.group(3);
  90             String amount = Objects.requireNonNull(m.group(4));
  91             String currencyPost = m.group(5);
  92
  93             String currency = null;
  94             if ((currencyPre != null) && (currencyPre.length() > 0)) {
  95                 if ((currencyPost != null) && (currencyPost.length() > 0))
  96                     return null;
  97                 currency = currencyPre;
  98             }
  99             else if ((currencyPost != null) && (currencyPost.length() > 0)) {
 100                 currency = currencyPost;
 101             }
 102
 103             amount = amount.replace(',', '.');
 104
 105             return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
 106         }
 107         else {
 108             return null;
 109         }
 110     }
 111     @Override
 112     protected void onProgressUpdate(Progress... values) {
 113         super.onProgressUpdate(values);
 114         Data.backgroundTaskProgress.postValue(values[0]);
 115     }
 116     @Override
 117     protected void onPostExecute(String error) {
 118         super.onPostExecute(error);
 119         Progress progress = new Progress();
 120         progress.setState(ProgressState.FINISHED);
 121         progress.setError(error);
 122         onProgressUpdate(progress);
 123     }
 124     @Override
 125     protected void onCancelled() {
 126         super.onCancelled();
 127         Progress progress = new Progress();
 128         progress.setState(ProgressState.FINISHED);
 129         onProgressUpdate(progress);
 130     }
 131     private String retrieveTransactionListLegacy() throws IOException, HTTPException {
 132         Progress progress = Progress.indeterminate();
 133         progress.setState(ProgressState.RUNNING);
 134         int maxTransactionId = -1;
 135         ArrayList<LedgerAccount> list = new ArrayList<>();
 136         HashMap<String, LedgerAccount> map = new HashMap<>();
 137         ArrayList<LedgerAccount> displayed = new ArrayList<>();
 138         ArrayList<LedgerTransaction> transactions = new ArrayList<>();
 139         LedgerAccount lastAccount = null;
 140         ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
 141
 142         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
 143         http.setAllowUserInteraction(false);
 144         publishProgress(progress);
 145         if (http.getResponseCode() != 200)
 146             throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 147
 148         try (InputStream resp = http.getInputStream()) {
 149             if (http.getResponseCode() != 200)
 150                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 151
 152             int matchedTransactionsCount = 0;
 153
 154             ParserState state = ParserState.EXPECTING_ACCOUNT;
 155             String line;
 156             BufferedReader buf =
 157                     new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
 158
 159             int processedTransactionCount = 0;
 160             int transactionId = 0;
 161             LedgerTransaction transaction = null;
 162             LINES:
 163             while ((line = buf.readLine()) != null) {
 164                 throwIfCancelled();
 165                 Matcher m;
 166                 m = reComment.matcher(line);
 167                 if (m.find()) {
 168                     // TODO: comments are ignored for now
 169 //                            Log.v("transaction-parser", "Ignoring comment");
 170                     continue;
 171                 }
 172                 //L(String.format("State is %d", updating));
 173                 switch (state) {
 174                     case EXPECTING_ACCOUNT:
 175                         if (line.equals("<h2>General Journal</h2>")) {
 176                             state = ParserState.EXPECTING_TRANSACTION;
 177                             L("→ expecting transaction");
 178                             continue;
 179                         }
 180                         m = reAccountName.matcher(line);
 181                         if (m.find()) {
 182                             String acct_encoded = m.group(1);
 183                             String accName = URLDecoder.decode(acct_encoded, "UTF-8");
 184                             accName = accName.replace("\"", "");
 185                             L(String.format("found account: %s", accName));
 186
 187                             lastAccount = map.get(accName);
 188                             if (lastAccount != null) {
 189                                 L(String.format("ignoring duplicate account '%s'", accName));
 190                                 continue;
 191                             }
 192                             String parentAccountName = LedgerAccount.extractParentName(accName);
 193                             LedgerAccount parentAccount;
 194                             if (parentAccountName != null) {
 195                                 parentAccount = ensureAccountExists(parentAccountName, map,
 196                                         syntheticAccounts);
 197                             }
 198                             else {
 199                                 parentAccount = null;
 200                             }
 201                             lastAccount = new LedgerAccount(profile, accName, parentAccount);
 202
 203                             list.add(lastAccount);
 204                             map.put(accName, lastAccount);
 205
 206                             state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
 207                             L("→ expecting account amount");
 208                         }
 209                         break;
 210
 211                     case EXPECTING_ACCOUNT_AMOUNT:
 212                         m = reAccountValue.matcher(line);
 213                         boolean match_found = false;
 214                         while (m.find()) {
 215                             throwIfCancelled();
 216
 217                             match_found = true;
 218                             String value = Objects.requireNonNull(m.group(1));
 219                             String currency = m.group(2);
 220                             if (currency == null)
 221                                 currency = "";
 222
 223                             {
 224                                 Matcher tmpM = reDecimalComma.matcher(value);
 225                                 if (tmpM.find()) {
 226                                     value = value.replace(".", "");
 227                                     value = value.replace(',', '.');
 228                                 }
 229
 230                                 tmpM = reDecimalPoint.matcher(value);
 231                                 if (tmpM.find()) {
 232                                     value = value.replace(",", "");
 233                                     value = value.replace(" ", "");
 234                                 }
 235                             }
 236                             L("curr=" + currency + ", value=" + value);
 237                             final float val = Float.parseFloat(value);
 238                             lastAccount.addAmount(val, currency);
 239                             for (LedgerAccount syn : syntheticAccounts) {
 240                                 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
 241                                         currency, val, syn.getName()));
 242                                 syn.addAmount(val, currency);
 243                             }
 244                         }
 245
 246                         if (match_found) {
 247                             syntheticAccounts.clear();
 248                             state = ParserState.EXPECTING_ACCOUNT;
 249                             L("→ expecting account");
 250                         }
 251
 252                         break;
 253
 254                     case EXPECTING_TRANSACTION:
 255                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 256                             continue;
 257                         m = reTransactionStart.matcher(line);
 258                         if (m.find()) {
 259                             transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
 260                             state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
 261                             L(String.format(Locale.ENGLISH,
 262                                     "found transaction %d → expecting description", transactionId));
 263                             progress.setProgress(++processedTransactionCount);
 264                             if (maxTransactionId < transactionId)
 265                                 maxTransactionId = transactionId;
 266                             if ((progress.isIndeterminate()) ||
 267                                 (progress.getTotal() < transactionId))
 268                                 progress.setTotal(transactionId);
 269                             publishProgress(progress);
 270                         }
 271                         m = reEnd.matcher(line);
 272                         if (m.find()) {
 273                             L("--- transaction value complete ---");
 274                             break LINES;
 275                         }
 276                         break;
 277
 278                     case EXPECTING_TRANSACTION_DESCRIPTION:
 279                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 280                             continue;
 281                         m = reTransactionDescription.matcher(line);
 282                         if (m.find()) {
 283                             if (transactionId == 0)
 284                                 throw new TransactionParserException(
 285                                         "Transaction Id is 0 while expecting " + "description");
 286
 287                             String date = Objects.requireNonNull(m.group(1));
 288                             try {
 289                                 int equalsIndex = date.indexOf('=');
 290                                 if (equalsIndex >= 0)
 291                                     date = date.substring(equalsIndex + 1);
 292                                 transaction =
 293                                         new LedgerTransaction(transactionId, date, m.group(2));
 294                             }
 295                             catch (ParseException e) {
 296                                 e.printStackTrace();
 297                                 return String.format("Error parsing date '%s'", date);
 298                             }
 299                             state = ParserState.EXPECTING_TRANSACTION_DETAILS;
 300                             L(String.format(Locale.ENGLISH,
 301                                     "transaction %d created for %s (%s) →" + " expecting details",
 302                                     transactionId, date, m.group(2)));
 303                         }
 304                         break;
 305
 306                     case EXPECTING_TRANSACTION_DETAILS:
 307                         if (line.isEmpty()) {
 308                             // transaction data collected
 309
 310                             transaction.finishLoading();
 311                             transactions.add(transaction);
 312
 313                             state = ParserState.EXPECTING_TRANSACTION;
 314                             L(String.format("transaction %s parsed → expecting transaction",
 315                                     transaction.getId()));
 316
 317 // sounds like a good idea, but transaction-1 may not be the first one chronologically
 318 // for example, when you add the initial seeding transaction after entering some others
 319 //                                            if (transactionId == 1) {
 320 //                                                L("This was the initial transaction.
 321 //                                                Terminating " +
 322 //                                                  "parser");
 323 //                                                break LINES;
 324 //                                            }
 325                         }
 326                         else {
 327                             LedgerTransactionAccount lta = parseTransactionAccountLine(line);
 328                             if (lta != null) {
 329                                 transaction.addAccount(lta);
 330                                 L(String.format(Locale.ENGLISH, "%d: %s = %s", transaction.getId(),
 331                                         lta.getAccountName(), lta.getAmount()));
 332                             }
 333                             else
 334                                 throw new IllegalStateException(
 335                                         String.format("Can't parse transaction %d details: %s",
 336                                                 transactionId, line));
 337                         }
 338                         break;
 339                     default:
 340                         throw new RuntimeException(
 341                                 String.format("Unknown parser updating %s", state.name()));
 342                 }
 343             }
 344
 345             throwIfCancelled();
 346
 347             profile.setAndStoreAccountAndTransactionListFromWeb(list, transactions);
 348
 349             return null;
 350         }
 351     }
 352     private @NonNull
 353     LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
 354                                       ArrayList<LedgerAccount> createdAccounts) {
 355         LedgerAccount acc = map.get(accountName);
 356
 357         if (acc != null)
 358             return acc;
 359
 360         String parentName = LedgerAccount.extractParentName(accountName);
 361         LedgerAccount parentAccount;
 362         if (parentName != null) {
 363             parentAccount = ensureAccountExists(parentName, map, createdAccounts);
 364         }
 365         else {
 366             parentAccount = null;
 367         }
 368
 369         acc = new LedgerAccount(profile, accountName, parentAccount);
 370         createdAccounts.add(acc);
 371         return acc;
 372     }
 373     private boolean retrieveAccountList() throws IOException, HTTPException {
 374         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
 375         http.setAllowUserInteraction(false);
 376         switch (http.getResponseCode()) {
 377             case 200:
 378                 break;
 379             case 404:
 380                 return false;
 381             default:
 382                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 383         }
 384         publishProgress(Progress.indeterminate());
 385         SQLiteDatabase db = App.getDatabase();
 386         ArrayList<LedgerAccount> list = new ArrayList<>();
 387         HashMap<String, LedgerAccount> map = new HashMap<>();
 388         HashMap<String, LedgerAccount> currentMap = new HashMap<>();
 389         for (LedgerAccount acc : Objects.requireNonNull(profile.getAllAccounts()))
 390             currentMap.put(acc.getName(), acc);
 391         try (InputStream resp = http.getInputStream()) {
 392             if (http.getResponseCode() != 200)
 393                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 394
 395             AccountListParser parser = new AccountListParser(resp);
 396
 397             while (true) {
 398                 throwIfCancelled();
 399                 ParsedLedgerAccount parsedAccount = parser.nextAccount();
 400                 if (parsedAccount == null) {
 401                     break;
 402                 }
 403
 404                 final String accName = parsedAccount.getAname();
 405                 LedgerAccount acc = map.get(accName);
 406                 if (acc != null)
 407                     throw new RuntimeException(
 408                             String.format("Account '%s' already present", acc.getName()));
 409                 String parentName = LedgerAccount.extractParentName(accName);
 410                 ArrayList<LedgerAccount> createdParents = new ArrayList<>();
 411                 LedgerAccount parent;
 412                 if (parentName == null) {
 413                     parent = null;
 414                 }
 415                 else {
 416                     parent = ensureAccountExists(parentName, map, createdParents);
 417                     parent.setHasSubAccounts(true);
 418                 }
 419                 acc = new LedgerAccount(profile, accName, parent);
 420                 list.add(acc);
 421                 map.put(accName, acc);
 422
 423                 String lastCurrency = null;
 424                 float lastCurrencyAmount = 0;
 425                 for (ParsedBalance b : parsedAccount.getAibalance()) {
 426                     throwIfCancelled();
 427                     final String currency = b.getAcommodity();
 428                     final float amount = b.getAquantity()
 429                                           .asFloat();
 430                     if (currency.equals(lastCurrency)) {
 431                         lastCurrencyAmount += amount;
 432                     }
 433                     else {
 434                         if (lastCurrency != null) {
 435                             acc.addAmount(lastCurrencyAmount, lastCurrency);
 436                         }
 437                         lastCurrency = currency;
 438                         lastCurrencyAmount = amount;
 439                     }
 440                 }
 441                 if (lastCurrency != null) {
 442                     acc.addAmount(lastCurrencyAmount, lastCurrency);
 443                 }
 444                 for (LedgerAccount p : createdParents)
 445                     acc.propagateAmountsTo(p);
 446             }
 447             throwIfCancelled();
 448         }
 449
 450         // the current account tree may have changed, update the new-to be tree to match
 451         for (LedgerAccount acc : list) {
 452             LedgerAccount prevData = currentMap.get(acc.getName());
 453             if (prevData != null) {
 454                 acc.setExpanded(prevData.isExpanded());
 455                 acc.setAmountsExpanded(prevData.amountsExpanded());
 456             }
 457         }
 458
 459         profile.setAndStoreAccountListFromWeb(list);
 460         return true;
 461     }
 462     private boolean retrieveTransactionList() throws IOException, ParseException, HTTPException {
 463         Progress progress = new Progress();
 464         int maxTransactionId = Data.transactions.size();
 465
 466         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
 467         http.setAllowUserInteraction(false);
 468         publishProgress(progress);
 469         switch (http.getResponseCode()) {
 470             case 200:
 471                 break;
 472             case 404:
 473                 return false;
 474             default:
 475                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 476         }
 477         try (InputStream resp = http.getInputStream()) {
 478             throwIfCancelled();
 479             ArrayList<LedgerTransaction> trList = new ArrayList<>();
 480
 481             TransactionListParser parser = new TransactionListParser(resp);
 482
 483             int processedTransactionCount = 0;
 484
 485             while (true) {
 486                 throwIfCancelled();
 487                 ParsedLedgerTransaction parsedTransaction = parser.nextTransaction();
 488                 throwIfCancelled();
 489                 if (parsedTransaction == null)
 490                     break;
 491
 492                 LedgerTransaction transaction = parsedTransaction.asLedgerTransaction();
 493                 trList.add(transaction);
 494
 495                 progress.setProgress(++processedTransactionCount);
 496                 publishProgress(progress);
 497             }
 498
 499             throwIfCancelled();
 500             profile.setAndStoreTransactionList(trList);
 501         }
 502
 503         return true;
 504     }
 505
 506     @SuppressLint("DefaultLocale")
 507     @Override
 508     protected String doInBackground(Void... params) {
 509         Data.backgroundTaskStarted();
 510         try {
 511             if (!retrieveAccountList() || !retrieveTransactionList())
 512                 return retrieveTransactionListLegacy();
 513             return null;
 514         }
 515         catch (MalformedURLException e) {
 516             e.printStackTrace();
 517             return "Invalid server URL";
 518         }
 519         catch (HTTPException e) {
 520             e.printStackTrace();
 521             return String.format("HTTP error %d: %s", e.getResponseCode(), e.getResponseMessage());
 522         }
 523         catch (IOException e) {
 524             e.printStackTrace();
 525             return e.getLocalizedMessage();
 526         }
 527         catch (ParseException e) {
 528             e.printStackTrace();
 529             return "Network error";
 530         }
 531         catch (OperationCanceledException e) {
 532             e.printStackTrace();
 533             return "Operation cancelled";
 534         }
 535         finally {
 536             Data.backgroundTaskFinished();
 537         }
 538     }
 539     private void throwIfCancelled() {
 540         if (isCancelled())
 541             throw new OperationCanceledException(null);
 542     }
 543     private enum ParserState {
 544         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
 545         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
 546     }
 547
 548     public enum ProgressState {STARTING, RUNNING, FINISHED}
 549
 550     public static class Progress {
 551         private int progress;
 552         private int total;
 553         private ProgressState state = ProgressState.RUNNING;
 554         private String error = null;
 555         private boolean indeterminate;
 556         Progress() {
 557             indeterminate = true;
 558         }
 559         Progress(int progress, int total) {
 560             this.indeterminate = false;
 561             this.progress = progress;
 562             this.total = total;
 563         }
 564         public static Progress indeterminate() {
 565             return new Progress();
 566         }
 567         public static Progress finished(String error) {
 568             Progress p = new Progress();
 569             p.setState(ProgressState.FINISHED);
 570             p.setError(error);
 571             return p;
 572         }
 573         public int getProgress() {
 574             ensureState(ProgressState.RUNNING);
 575             return progress;
 576         }
 577         protected void setProgress(int progress) {
 578             this.progress = progress;
 579             this.state = ProgressState.RUNNING;
 580         }
 581         public int getTotal() {
 582             ensureState(ProgressState.RUNNING);
 583             return total;
 584         }
 585         protected void setTotal(int total) {
 586             this.total = total;
 587             state = ProgressState.RUNNING;
 588             indeterminate = total == -1;
 589         }
 590         private void ensureState(ProgressState wanted) {
 591             if (state != wanted)
 592                 throw new IllegalStateException(
 593                         String.format("Bad state: %s, expected %s", state, wanted));
 594         }
 595         public ProgressState getState() {
 596             return state;
 597         }
 598         public void setState(ProgressState state) {
 599             this.state = state;
 600         }
 601         public String getError() {
 602             ensureState(ProgressState.FINISHED);
 603             return error;
 604         }
 605         public void setError(String error) {
 606             this.error = error;
 607             state = ProgressState.FINISHED;
 608         }
 609         public boolean isIndeterminate() {
 610             return indeterminate;
 611         }
 612         public void setIndeterminate(boolean indeterminate) {
 613             this.indeterminate = indeterminate;
 614         }
 615     }
 616
 617     private static class TransactionParserException extends IllegalStateException {
 618         TransactionParserException(String message) {
 619             super(message);
 620         }
 621     }
 622 }