app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java

   1 /*
   2  * Copyright © 2020 Damyan Ivanov.
   3  * This file is part of MoLe.
   4  * MoLe is free software: you can distribute it and/or modify it
   5  * under the term of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your opinion), any later version.
   8  *
   9  * MoLe is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12  * GNU General Public License terms for details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
  16  */
  17
  18 package net.ktnx.mobileledger.async;
  19
  20 import android.annotation.SuppressLint;
  21 import android.database.sqlite.SQLiteDatabase;
  22 import android.os.AsyncTask;
  23 import android.os.OperationCanceledException;
  24
  25 import androidx.annotation.NonNull;
  26
  27 import net.ktnx.mobileledger.App;
  28 import net.ktnx.mobileledger.err.HTTPException;
  29 import net.ktnx.mobileledger.json.v1_15.AccountListParser;
  30 import net.ktnx.mobileledger.json.v1_15.ParsedBalance;
  31 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerAccount;
  32 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerTransaction;
  33 import net.ktnx.mobileledger.json.v1_15.TransactionListParser;
  34 import net.ktnx.mobileledger.model.Data;
  35 import net.ktnx.mobileledger.model.LedgerAccount;
  36 import net.ktnx.mobileledger.model.LedgerTransaction;
  37 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
  38 import net.ktnx.mobileledger.model.MobileLedgerProfile;
  39 import net.ktnx.mobileledger.utils.NetworkUtil;
  40
  41 import java.io.BufferedReader;
  42 import java.io.IOException;
  43 import java.io.InputStream;
  44 import java.io.InputStreamReader;
  45 import java.net.HttpURLConnection;
  46 import java.net.MalformedURLException;
  47 import java.net.URLDecoder;
  48 import java.nio.charset.StandardCharsets;
  49 import java.text.ParseException;
  50 import java.util.ArrayList;
  51 import java.util.HashMap;
  52 import java.util.Locale;
  53 import java.util.Objects;
  54 import java.util.regex.Matcher;
  55 import java.util.regex.Pattern;
  56
  57
  58 public class RetrieveTransactionsTask
  59         extends AsyncTask<Void, RetrieveTransactionsTask.Progress, String> {
  60     private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
  61     private static final Pattern reComment = Pattern.compile("^\\s*;");
  62     private static final Pattern reTransactionStart = Pattern.compile(
  63             "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
  64             "\"[^\"]*>([\\d.-]+)</td>");
  65     private static final Pattern reTransactionDescription =
  66             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
  67     private static final Pattern reTransactionDetails = Pattern.compile(
  68             "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
  69             "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
  70     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
  71     private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
  72     private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
  73     // %3A is '='
  74     private Pattern reAccountName = Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
  75     private Pattern reAccountValue = Pattern.compile(
  76             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
  77     private MobileLedgerProfile profile;
  78     private int expectedPostingsCount = -1;
  79     public RetrieveTransactionsTask(@NonNull MobileLedgerProfile profile) {
  80         this.profile = profile;
  81     }
  82     private static void L(String msg) {
  83         //debug("transaction-parser", msg);
  84     }
  85     static LedgerTransactionAccount parseTransactionAccountLine(String line) {
  86         Matcher m = reTransactionDetails.matcher(line);
  87         if (m.find()) {
  88             String postingStatus = m.group(1);
  89             String acc_name = m.group(2);
  90             String currencyPre = m.group(3);
  91             String amount = Objects.requireNonNull(m.group(4));
  92             String currencyPost = m.group(5);
  93
  94             String currency = null;
  95             if ((currencyPre != null) && (currencyPre.length() > 0)) {
  96                 if ((currencyPost != null) && (currencyPost.length() > 0))
  97                     return null;
  98                 currency = currencyPre;
  99             }
 100             else if ((currencyPost != null) && (currencyPost.length() > 0)) {
 101                 currency = currencyPost;
 102             }
 103
 104             amount = amount.replace(',', '.');
 105
 106             return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
 107         }
 108         else {
 109             return null;
 110         }
 111     }
 112     @Override
 113     protected void onProgressUpdate(Progress... values) {
 114         super.onProgressUpdate(values);
 115         Data.backgroundTaskProgress.postValue(values[0]);
 116     }
 117     @Override
 118     protected void onPostExecute(String error) {
 119         super.onPostExecute(error);
 120         Progress progress = new Progress();
 121         progress.setState(ProgressState.FINISHED);
 122         progress.setError(error);
 123         onProgressUpdate(progress);
 124     }
 125     @Override
 126     protected void onCancelled() {
 127         super.onCancelled();
 128         Progress progress = new Progress();
 129         progress.setState(ProgressState.FINISHED);
 130         onProgressUpdate(progress);
 131     }
 132     private String retrieveTransactionListLegacy() throws IOException, HTTPException {
 133         Progress progress = Progress.indeterminate();
 134         progress.setState(ProgressState.RUNNING);
 135         progress.setTotal(expectedPostingsCount);
 136         int maxTransactionId = -1;
 137         ArrayList<LedgerAccount> list = new ArrayList<>();
 138         HashMap<String, LedgerAccount> map = new HashMap<>();
 139         ArrayList<LedgerAccount> displayed = new ArrayList<>();
 140         ArrayList<LedgerTransaction> transactions = new ArrayList<>();
 141         LedgerAccount lastAccount = null;
 142         ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
 143
 144         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
 145         http.setAllowUserInteraction(false);
 146         publishProgress(progress);
 147         if (http.getResponseCode() != 200)
 148             throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 149
 150         try (InputStream resp = http.getInputStream()) {
 151             if (http.getResponseCode() != 200)
 152                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 153
 154             int matchedTransactionsCount = 0;
 155
 156             ParserState state = ParserState.EXPECTING_ACCOUNT;
 157             String line;
 158             BufferedReader buf =
 159                     new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
 160
 161             int processedTransactionCount = 0;
 162             int transactionId = 0;
 163             LedgerTransaction transaction = null;
 164             LINES:
 165             while ((line = buf.readLine()) != null) {
 166                 throwIfCancelled();
 167                 Matcher m;
 168                 m = reComment.matcher(line);
 169                 if (m.find()) {
 170                     // TODO: comments are ignored for now
 171 //                            Log.v("transaction-parser", "Ignoring comment");
 172                     continue;
 173                 }
 174                 //L(String.format("State is %d", updating));
 175                 switch (state) {
 176                     case EXPECTING_ACCOUNT:
 177                         if (line.equals("<h2>General Journal</h2>")) {
 178                             state = ParserState.EXPECTING_TRANSACTION;
 179                             L("→ expecting transaction");
 180                             continue;
 181                         }
 182                         m = reAccountName.matcher(line);
 183                         if (m.find()) {
 184                             String acct_encoded = m.group(1);
 185                             String accName = URLDecoder.decode(acct_encoded, "UTF-8");
 186                             accName = accName.replace("\"", "");
 187                             L(String.format("found account: %s", accName));
 188
 189                             lastAccount = map.get(accName);
 190                             if (lastAccount != null) {
 191                                 L(String.format("ignoring duplicate account '%s'", accName));
 192                                 continue;
 193                             }
 194                             String parentAccountName = LedgerAccount.extractParentName(accName);
 195                             LedgerAccount parentAccount;
 196                             if (parentAccountName != null) {
 197                                 parentAccount = ensureAccountExists(parentAccountName, map,
 198                                         syntheticAccounts);
 199                             }
 200                             else {
 201                                 parentAccount = null;
 202                             }
 203                             lastAccount = new LedgerAccount(profile, accName, parentAccount);
 204
 205                             list.add(lastAccount);
 206                             map.put(accName, lastAccount);
 207
 208                             state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
 209                             L("→ expecting account amount");
 210                         }
 211                         break;
 212
 213                     case EXPECTING_ACCOUNT_AMOUNT:
 214                         m = reAccountValue.matcher(line);
 215                         boolean match_found = false;
 216                         while (m.find()) {
 217                             throwIfCancelled();
 218
 219                             match_found = true;
 220                             String value = Objects.requireNonNull(m.group(1));
 221                             String currency = m.group(2);
 222                             if (currency == null)
 223                                 currency = "";
 224
 225                             {
 226                                 Matcher tmpM = reDecimalComma.matcher(value);
 227                                 if (tmpM.find()) {
 228                                     value = value.replace(".", "");
 229                                     value = value.replace(',', '.');
 230                                 }
 231
 232                                 tmpM = reDecimalPoint.matcher(value);
 233                                 if (tmpM.find()) {
 234                                     value = value.replace(",", "");
 235                                     value = value.replace(" ", "");
 236                                 }
 237                             }
 238                             L("curr=" + currency + ", value=" + value);
 239                             final float val = Float.parseFloat(value);
 240                             lastAccount.addAmount(val, currency);
 241                             for (LedgerAccount syn : syntheticAccounts) {
 242                                 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
 243                                         currency, val, syn.getName()));
 244                                 syn.addAmount(val, currency);
 245                             }
 246                         }
 247
 248                         if (match_found) {
 249                             syntheticAccounts.clear();
 250                             state = ParserState.EXPECTING_ACCOUNT;
 251                             L("→ expecting account");
 252                         }
 253
 254                         break;
 255
 256                     case EXPECTING_TRANSACTION:
 257                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 258                             continue;
 259                         m = reTransactionStart.matcher(line);
 260                         if (m.find()) {
 261                             transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
 262                             state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
 263                             L(String.format(Locale.ENGLISH,
 264                                     "found transaction %d → expecting description", transactionId));
 265                             progress.setProgress(++processedTransactionCount);
 266                             if (maxTransactionId < transactionId)
 267                                 maxTransactionId = transactionId;
 268                             if ((progress.isIndeterminate()) ||
 269                                 (progress.getTotal() < transactionId))
 270                                 progress.setTotal(transactionId);
 271                             publishProgress(progress);
 272                         }
 273                         m = reEnd.matcher(line);
 274                         if (m.find()) {
 275                             L("--- transaction value complete ---");
 276                             break LINES;
 277                         }
 278                         break;
 279
 280                     case EXPECTING_TRANSACTION_DESCRIPTION:
 281                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 282                             continue;
 283                         m = reTransactionDescription.matcher(line);
 284                         if (m.find()) {
 285                             if (transactionId == 0)
 286                                 throw new TransactionParserException(
 287                                         "Transaction Id is 0 while expecting " + "description");
 288
 289                             String date = Objects.requireNonNull(m.group(1));
 290                             try {
 291                                 int equalsIndex = date.indexOf('=');
 292                                 if (equalsIndex >= 0)
 293                                     date = date.substring(equalsIndex + 1);
 294                                 transaction =
 295                                         new LedgerTransaction(transactionId, date, m.group(2));
 296                             }
 297                             catch (ParseException e) {
 298                                 e.printStackTrace();
 299                                 return String.format("Error parsing date '%s'", date);
 300                             }
 301                             state = ParserState.EXPECTING_TRANSACTION_DETAILS;
 302                             L(String.format(Locale.ENGLISH,
 303                                     "transaction %d created for %s (%s) →" + " expecting details",
 304                                     transactionId, date, m.group(2)));
 305                         }
 306                         break;
 307
 308                     case EXPECTING_TRANSACTION_DETAILS:
 309                         if (line.isEmpty()) {
 310                             // transaction data collected
 311
 312                             transaction.finishLoading();
 313                             transactions.add(transaction);
 314
 315                             state = ParserState.EXPECTING_TRANSACTION;
 316                             L(String.format("transaction %s parsed → expecting transaction",
 317                                     transaction.getId()));
 318
 319 // sounds like a good idea, but transaction-1 may not be the first one chronologically
 320 // for example, when you add the initial seeding transaction after entering some others
 321 //                                            if (transactionId == 1) {
 322 //                                                L("This was the initial transaction.
 323 //                                                Terminating " +
 324 //                                                  "parser");
 325 //                                                break LINES;
 326 //                                            }
 327                         }
 328                         else {
 329                             LedgerTransactionAccount lta = parseTransactionAccountLine(line);
 330                             if (lta != null) {
 331                                 transaction.addAccount(lta);
 332                                 L(String.format(Locale.ENGLISH, "%d: %s = %s", transaction.getId(),
 333                                         lta.getAccountName(), lta.getAmount()));
 334                             }
 335                             else
 336                                 throw new IllegalStateException(
 337                                         String.format("Can't parse transaction %d details: %s",
 338                                                 transactionId, line));
 339                         }
 340                         break;
 341                     default:
 342                         throw new RuntimeException(
 343                                 String.format("Unknown parser updating %s", state.name()));
 344                 }
 345             }
 346
 347             throwIfCancelled();
 348
 349             profile.setAndStoreAccountAndTransactionListFromWeb(list, transactions);
 350
 351             return null;
 352         }
 353     }
 354     private @NonNull
 355     LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
 356                                       ArrayList<LedgerAccount> createdAccounts) {
 357         LedgerAccount acc = map.get(accountName);
 358
 359         if (acc != null)
 360             return acc;
 361
 362         String parentName = LedgerAccount.extractParentName(accountName);
 363         LedgerAccount parentAccount;
 364         if (parentName != null) {
 365             parentAccount = ensureAccountExists(parentName, map, createdAccounts);
 366         }
 367         else {
 368             parentAccount = null;
 369         }
 370
 371         acc = new LedgerAccount(profile, accountName, parentAccount);
 372         createdAccounts.add(acc);
 373         return acc;
 374     }
 375     private boolean retrieveAccountList() throws IOException, HTTPException {
 376         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
 377         http.setAllowUserInteraction(false);
 378         switch (http.getResponseCode()) {
 379             case 200:
 380                 break;
 381             case 404:
 382                 return false;
 383             default:
 384                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 385         }
 386         publishProgress(Progress.indeterminate());
 387         SQLiteDatabase db = App.getDatabase();
 388         ArrayList<LedgerAccount> list = new ArrayList<>();
 389         HashMap<String, LedgerAccount> map = new HashMap<>();
 390         HashMap<String, LedgerAccount> currentMap = new HashMap<>();
 391         for (LedgerAccount acc : Objects.requireNonNull(profile.getAllAccounts()))
 392             currentMap.put(acc.getName(), acc);
 393         try (InputStream resp = http.getInputStream()) {
 394             if (http.getResponseCode() != 200)
 395                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 396
 397             AccountListParser parser = new AccountListParser(resp);
 398             expectedPostingsCount = 0;
 399
 400             while (true) {
 401                 throwIfCancelled();
 402                 ParsedLedgerAccount parsedAccount = parser.nextAccount();
 403                 if (parsedAccount == null) {
 404                     break;
 405                 }
 406                 expectedPostingsCount += parsedAccount.getAnumpostings();
 407                 final String accName = parsedAccount.getAname();
 408                 LedgerAccount acc = map.get(accName);
 409                 if (acc != null)
 410                     throw new RuntimeException(
 411                             String.format("Account '%s' already present", acc.getName()));
 412                 String parentName = LedgerAccount.extractParentName(accName);
 413                 ArrayList<LedgerAccount> createdParents = new ArrayList<>();
 414                 LedgerAccount parent;
 415                 if (parentName == null) {
 416                     parent = null;
 417                 }
 418                 else {
 419                     parent = ensureAccountExists(parentName, map, createdParents);
 420                     parent.setHasSubAccounts(true);
 421                 }
 422                 acc = new LedgerAccount(profile, accName, parent);
 423                 list.add(acc);
 424                 map.put(accName, acc);
 425
 426                 String lastCurrency = null;
 427                 float lastCurrencyAmount = 0;
 428                 for (ParsedBalance b : parsedAccount.getAibalance()) {
 429                     throwIfCancelled();
 430                     final String currency = b.getAcommodity();
 431                     final float amount = b.getAquantity()
 432                                           .asFloat();
 433                     if (currency.equals(lastCurrency)) {
 434                         lastCurrencyAmount += amount;
 435                     }
 436                     else {
 437                         if (lastCurrency != null) {
 438                             acc.addAmount(lastCurrencyAmount, lastCurrency);
 439                         }
 440                         lastCurrency = currency;
 441                         lastCurrencyAmount = amount;
 442                     }
 443                 }
 444                 if (lastCurrency != null) {
 445                     acc.addAmount(lastCurrencyAmount, lastCurrency);
 446                 }
 447                 for (LedgerAccount p : createdParents)
 448                     acc.propagateAmountsTo(p);
 449             }
 450             throwIfCancelled();
 451         }
 452
 453         // the current account tree may have changed, update the new-to be tree to match
 454         for (LedgerAccount acc : list) {
 455             LedgerAccount prevData = currentMap.get(acc.getName());
 456             if (prevData != null) {
 457                 acc.setExpanded(prevData.isExpanded());
 458                 acc.setAmountsExpanded(prevData.amountsExpanded());
 459             }
 460         }
 461
 462         profile.setAndStoreAccountListFromWeb(list);
 463         return true;
 464     }
 465     private boolean retrieveTransactionList() throws IOException, ParseException, HTTPException {
 466         Progress progress = new Progress();
 467         int maxTransactionId = Data.transactions.size();
 468         progress.setTotal(expectedPostingsCount);
 469
 470         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
 471         http.setAllowUserInteraction(false);
 472         publishProgress(progress);
 473         switch (http.getResponseCode()) {
 474             case 200:
 475                 break;
 476             case 404:
 477                 return false;
 478             default:
 479                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 480         }
 481         try (InputStream resp = http.getInputStream()) {
 482             throwIfCancelled();
 483             ArrayList<LedgerTransaction> trList = new ArrayList<>();
 484
 485             TransactionListParser parser = new TransactionListParser(resp);
 486
 487             int processedPostings = 0;
 488
 489             while (true) {
 490                 throwIfCancelled();
 491                 ParsedLedgerTransaction parsedTransaction = parser.nextTransaction();
 492                 throwIfCancelled();
 493                 if (parsedTransaction == null)
 494                     break;
 495
 496                 LedgerTransaction transaction = parsedTransaction.asLedgerTransaction();
 497                 trList.add(transaction);
 498
 499                 progress.setProgress(processedPostings += transaction.getAccounts()
 500                                                                      .size());
 501                 publishProgress(progress);
 502             }
 503
 504             throwIfCancelled();
 505             profile.setAndStoreTransactionList(trList);
 506         }
 507
 508         return true;
 509     }
 510
 511     @SuppressLint("DefaultLocale")
 512     @Override
 513     protected String doInBackground(Void... params) {
 514         Data.backgroundTaskStarted();
 515         try {
 516             if (!retrieveAccountList() || !retrieveTransactionList())
 517                 return retrieveTransactionListLegacy();
 518             return null;
 519         }
 520         catch (MalformedURLException e) {
 521             e.printStackTrace();
 522             return "Invalid server URL";
 523         }
 524         catch (HTTPException e) {
 525             e.printStackTrace();
 526             return String.format("HTTP error %d: %s", e.getResponseCode(), e.getResponseMessage());
 527         }
 528         catch (IOException e) {
 529             e.printStackTrace();
 530             return e.getLocalizedMessage();
 531         }
 532         catch (ParseException e) {
 533             e.printStackTrace();
 534             return "Network error";
 535         }
 536         catch (OperationCanceledException e) {
 537             e.printStackTrace();
 538             return "Operation cancelled";
 539         }
 540         finally {
 541             Data.backgroundTaskFinished();
 542         }
 543     }
 544     private void throwIfCancelled() {
 545         if (isCancelled())
 546             throw new OperationCanceledException(null);
 547     }
 548     private enum ParserState {
 549         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
 550         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
 551     }
 552
 553     public enum ProgressState {STARTING, RUNNING, FINISHED}
 554
 555     public static class Progress {
 556         private int progress;
 557         private int total;
 558         private ProgressState state = ProgressState.RUNNING;
 559         private String error = null;
 560         private boolean indeterminate;
 561         Progress() {
 562             indeterminate = true;
 563         }
 564         Progress(int progress, int total) {
 565             this.indeterminate = false;
 566             this.progress = progress;
 567             this.total = total;
 568         }
 569         public static Progress indeterminate() {
 570             return new Progress();
 571         }
 572         public static Progress finished(String error) {
 573             Progress p = new Progress();
 574             p.setState(ProgressState.FINISHED);
 575             p.setError(error);
 576             return p;
 577         }
 578         public int getProgress() {
 579             ensureState(ProgressState.RUNNING);
 580             return progress;
 581         }
 582         protected void setProgress(int progress) {
 583             this.progress = progress;
 584             this.state = ProgressState.RUNNING;
 585         }
 586         public int getTotal() {
 587             ensureState(ProgressState.RUNNING);
 588             return total;
 589         }
 590         protected void setTotal(int total) {
 591             this.total = total;
 592             state = ProgressState.RUNNING;
 593             indeterminate = total == -1;
 594         }
 595         private void ensureState(ProgressState wanted) {
 596             if (state != wanted)
 597                 throw new IllegalStateException(
 598                         String.format("Bad state: %s, expected %s", state, wanted));
 599         }
 600         public ProgressState getState() {
 601             return state;
 602         }
 603         public void setState(ProgressState state) {
 604             this.state = state;
 605         }
 606         public String getError() {
 607             ensureState(ProgressState.FINISHED);
 608             return error;
 609         }
 610         public void setError(String error) {
 611             this.error = error;
 612             state = ProgressState.FINISHED;
 613         }
 614         public boolean isIndeterminate() {
 615             return indeterminate;
 616         }
 617         public void setIndeterminate(boolean indeterminate) {
 618             this.indeterminate = indeterminate;
 619         }
 620     }
 621
 622     private static class TransactionParserException extends IllegalStateException {
 623         TransactionParserException(String message) {
 624             super(message);
 625         }
 626     }
 627 }