app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java

   1 /*
   2  * Copyright © 2020 Damyan Ivanov.
   3  * This file is part of MoLe.
   4  * MoLe is free software: you can distribute it and/or modify it
   5  * under the term of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your opinion), any later version.
   8  *
   9  * MoLe is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12  * GNU General Public License terms for details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
  16  */
  17
  18 package net.ktnx.mobileledger.async;
  19
  20 import android.annotation.SuppressLint;
  21 import android.database.sqlite.SQLiteDatabase;
  22 import android.os.AsyncTask;
  23 import android.os.OperationCanceledException;
  24
  25 import androidx.annotation.NonNull;
  26
  27 import net.ktnx.mobileledger.App;
  28 import net.ktnx.mobileledger.err.HTTPException;
  29 import net.ktnx.mobileledger.json.v1_15.AccountListParser;
  30 import net.ktnx.mobileledger.json.v1_15.ParsedBalance;
  31 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerAccount;
  32 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerTransaction;
  33 import net.ktnx.mobileledger.json.v1_15.TransactionListParser;
  34 import net.ktnx.mobileledger.model.Data;
  35 import net.ktnx.mobileledger.model.LedgerAccount;
  36 import net.ktnx.mobileledger.model.LedgerTransaction;
  37 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
  38 import net.ktnx.mobileledger.model.MobileLedgerProfile;
  39 import net.ktnx.mobileledger.ui.activity.MainActivity;
  40 import net.ktnx.mobileledger.utils.NetworkUtil;
  41
  42 import java.io.BufferedReader;
  43 import java.io.IOException;
  44 import java.io.InputStream;
  45 import java.io.InputStreamReader;
  46 import java.lang.ref.WeakReference;
  47 import java.net.HttpURLConnection;
  48 import java.net.MalformedURLException;
  49 import java.net.URLDecoder;
  50 import java.nio.charset.StandardCharsets;
  51 import java.text.ParseException;
  52 import java.util.ArrayList;
  53 import java.util.HashMap;
  54 import java.util.Locale;
  55 import java.util.Objects;
  56 import java.util.regex.Matcher;
  57 import java.util.regex.Pattern;
  58
  59
  60 public class RetrieveTransactionsTask
  61         extends AsyncTask<Void, RetrieveTransactionsTask.Progress, String> {
  62     private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
  63     private static final Pattern reComment = Pattern.compile("^\\s*;");
  64     private static final Pattern reTransactionStart = Pattern.compile(
  65             "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
  66             "\"[^\"]*>([\\d.-]+)</td>");
  67     private static final Pattern reTransactionDescription =
  68             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
  69     private static final Pattern reTransactionDetails = Pattern.compile(
  70             "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
  71             "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
  72     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
  73     private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
  74     private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
  75     private WeakReference<MainActivity> contextRef;
  76     // %3A is '='
  77     private Pattern reAccountName = Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
  78     private Pattern reAccountValue = Pattern.compile(
  79             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
  80     private MobileLedgerProfile profile;
  81     public RetrieveTransactionsTask(WeakReference<MainActivity> contextRef,
  82                                     @NonNull MobileLedgerProfile profile) {
  83         this.contextRef = contextRef;
  84         this.profile = profile;
  85     }
  86     private static void L(String msg) {
  87         //debug("transaction-parser", msg);
  88     }
  89     static LedgerTransactionAccount parseTransactionAccountLine(String line) {
  90         Matcher m = reTransactionDetails.matcher(line);
  91         if (m.find()) {
  92             String postingStatus = m.group(1);
  93             String acc_name = m.group(2);
  94             String currencyPre = m.group(3);
  95             String amount = Objects.requireNonNull(m.group(4));
  96             String currencyPost = m.group(5);
  97
  98             String currency = null;
  99             if ((currencyPre != null) && (currencyPre.length() > 0)) {
 100                 if ((currencyPost != null) && (currencyPost.length() > 0))
 101                     return null;
 102                 currency = currencyPre;
 103             }
 104             else if ((currencyPost != null) && (currencyPost.length() > 0)) {
 105                 currency = currencyPost;
 106             }
 107
 108             amount = amount.replace(',', '.');
 109
 110             return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
 111         }
 112         else {
 113             return null;
 114         }
 115     }
 116     @Override
 117     protected void onProgressUpdate(Progress... values) {
 118         super.onProgressUpdate(values);
 119         MainActivity context = getContext();
 120         if (context == null)
 121             return;
 122         context.onRetrieveProgress(values[0]);
 123     }
 124     @Override
 125     protected void onPreExecute() {
 126         super.onPreExecute();
 127         MainActivity context = getContext();
 128         if (context == null)
 129             return;
 130         context.onRetrieveStart();
 131     }
 132     @Override
 133     protected void onPostExecute(String error) {
 134         super.onPostExecute(error);
 135         MainActivity context = getContext();
 136         if (context == null)
 137             return;
 138         context.onRetrieveDone(error);
 139     }
 140     @Override
 141     protected void onCancelled() {
 142         super.onCancelled();
 143         MainActivity context = getContext();
 144         if (context == null)
 145             return;
 146         context.onRetrieveDone(null);
 147     }
 148     private String retrieveTransactionListLegacy() throws IOException, HTTPException {
 149         Progress progress = new Progress();
 150         int maxTransactionId = Progress.INDETERMINATE;
 151         ArrayList<LedgerAccount> list = new ArrayList<>();
 152         HashMap<String, LedgerAccount> map = new HashMap<>();
 153         ArrayList<LedgerAccount> displayed = new ArrayList<>();
 154         ArrayList<LedgerTransaction> transactions = new ArrayList<>();
 155         LedgerAccount lastAccount = null;
 156         ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
 157
 158         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
 159         http.setAllowUserInteraction(false);
 160         publishProgress(progress);
 161         if (http.getResponseCode() != 200)
 162             throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 163
 164         try (InputStream resp = http.getInputStream()) {
 165             if (http.getResponseCode() != 200)
 166                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 167
 168             int matchedTransactionsCount = 0;
 169
 170             ParserState state = ParserState.EXPECTING_ACCOUNT;
 171             String line;
 172             BufferedReader buf =
 173                     new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
 174
 175             int processedTransactionCount = 0;
 176             int transactionId = 0;
 177             LedgerTransaction transaction = null;
 178             LINES:
 179             while ((line = buf.readLine()) != null) {
 180                 throwIfCancelled();
 181                 Matcher m;
 182                 m = reComment.matcher(line);
 183                 if (m.find()) {
 184                     // TODO: comments are ignored for now
 185 //                            Log.v("transaction-parser", "Ignoring comment");
 186                     continue;
 187                 }
 188                 //L(String.format("State is %d", updating));
 189                 switch (state) {
 190                     case EXPECTING_ACCOUNT:
 191                         if (line.equals("<h2>General Journal</h2>")) {
 192                             state = ParserState.EXPECTING_TRANSACTION;
 193                             L("→ expecting transaction");
 194                             continue;
 195                         }
 196                         m = reAccountName.matcher(line);
 197                         if (m.find()) {
 198                             String acct_encoded = m.group(1);
 199                             String accName = URLDecoder.decode(acct_encoded, "UTF-8");
 200                             accName = accName.replace("\"", "");
 201                             L(String.format("found account: %s", accName));
 202
 203                             lastAccount = map.get(accName);
 204                             if (lastAccount != null) {
 205                                 L(String.format("ignoring duplicate account '%s'", accName));
 206                                 continue;
 207                             }
 208                             String parentAccountName = LedgerAccount.extractParentName(accName);
 209                             LedgerAccount parentAccount;
 210                             if (parentAccountName != null) {
 211                                 parentAccount = ensureAccountExists(parentAccountName, map,
 212                                         syntheticAccounts);
 213                             }
 214                             else {
 215                                 parentAccount = null;
 216                             }
 217                             lastAccount = new LedgerAccount(profile, accName, parentAccount);
 218
 219                             list.add(lastAccount);
 220                             map.put(accName, lastAccount);
 221
 222                             state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
 223                             L("→ expecting account amount");
 224                         }
 225                         break;
 226
 227                     case EXPECTING_ACCOUNT_AMOUNT:
 228                         m = reAccountValue.matcher(line);
 229                         boolean match_found = false;
 230                         while (m.find()) {
 231                             throwIfCancelled();
 232
 233                             match_found = true;
 234                             String value = Objects.requireNonNull(m.group(1));
 235                             String currency = m.group(2);
 236                             if (currency == null)
 237                                 currency = "";
 238
 239                             {
 240                                 Matcher tmpM = reDecimalComma.matcher(value);
 241                                 if (tmpM.find()) {
 242                                     value = value.replace(".", "");
 243                                     value = value.replace(',', '.');
 244                                 }
 245
 246                                 tmpM = reDecimalPoint.matcher(value);
 247                                 if (tmpM.find()) {
 248                                     value = value.replace(",", "");
 249                                     value = value.replace(" ", "");
 250                                 }
 251                             }
 252                             L("curr=" + currency + ", value=" + value);
 253                             final float val = Float.parseFloat(value);
 254                             lastAccount.addAmount(val, currency);
 255                             for (LedgerAccount syn : syntheticAccounts) {
 256                                 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
 257                                         currency, val, syn.getName()));
 258                                 syn.addAmount(val, currency);
 259                             }
 260                         }
 261
 262                         if (match_found) {
 263                             syntheticAccounts.clear();
 264                             state = ParserState.EXPECTING_ACCOUNT;
 265                             L("→ expecting account");
 266                         }
 267
 268                         break;
 269
 270                     case EXPECTING_TRANSACTION:
 271                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 272                             continue;
 273                         m = reTransactionStart.matcher(line);
 274                         if (m.find()) {
 275                             transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
 276                             state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
 277                             L(String.format(Locale.ENGLISH,
 278                                     "found transaction %d → expecting description", transactionId));
 279                             progress.setProgress(++processedTransactionCount);
 280                             if (maxTransactionId < transactionId)
 281                                 maxTransactionId = transactionId;
 282                             if ((progress.getTotal() == Progress.INDETERMINATE) ||
 283                                 (progress.getTotal() < transactionId))
 284                                 progress.setTotal(transactionId);
 285                             publishProgress(progress);
 286                         }
 287                         m = reEnd.matcher(line);
 288                         if (m.find()) {
 289                             L("--- transaction value complete ---");
 290                             break LINES;
 291                         }
 292                         break;
 293
 294                     case EXPECTING_TRANSACTION_DESCRIPTION:
 295                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 296                             continue;
 297                         m = reTransactionDescription.matcher(line);
 298                         if (m.find()) {
 299                             if (transactionId == 0)
 300                                 throw new TransactionParserException(
 301                                         "Transaction Id is 0 while expecting " + "description");
 302
 303                             String date = Objects.requireNonNull(m.group(1));
 304                             try {
 305                                 int equalsIndex = date.indexOf('=');
 306                                 if (equalsIndex >= 0)
 307                                     date = date.substring(equalsIndex + 1);
 308                                 transaction =
 309                                         new LedgerTransaction(transactionId, date, m.group(2));
 310                             }
 311                             catch (ParseException e) {
 312                                 e.printStackTrace();
 313                                 return String.format("Error parsing date '%s'", date);
 314                             }
 315                             state = ParserState.EXPECTING_TRANSACTION_DETAILS;
 316                             L(String.format(Locale.ENGLISH,
 317                                     "transaction %d created for %s (%s) →" + " expecting details",
 318                                     transactionId, date, m.group(2)));
 319                         }
 320                         break;
 321
 322                     case EXPECTING_TRANSACTION_DETAILS:
 323                         if (line.isEmpty()) {
 324                             // transaction data collected
 325
 326                             transaction.finishLoading();
 327                             transactions.add(transaction);
 328
 329                             state = ParserState.EXPECTING_TRANSACTION;
 330                             L(String.format("transaction %s parsed → expecting transaction",
 331                                     transaction.getId()));
 332
 333 // sounds like a good idea, but transaction-1 may not be the first one chronologically
 334 // for example, when you add the initial seeding transaction after entering some others
 335 //                                            if (transactionId == 1) {
 336 //                                                L("This was the initial transaction.
 337 //                                                Terminating " +
 338 //                                                  "parser");
 339 //                                                break LINES;
 340 //                                            }
 341                         }
 342                         else {
 343                             LedgerTransactionAccount lta = parseTransactionAccountLine(line);
 344                             if (lta != null) {
 345                                 transaction.addAccount(lta);
 346                                 L(String.format(Locale.ENGLISH, "%d: %s = %s", transaction.getId(),
 347                                         lta.getAccountName(), lta.getAmount()));
 348                             }
 349                             else
 350                                 throw new IllegalStateException(
 351                                         String.format("Can't parse transaction %d details: %s",
 352                                                 transactionId, line));
 353                         }
 354                         break;
 355                     default:
 356                         throw new RuntimeException(
 357                                 String.format("Unknown parser updating %s", state.name()));
 358                 }
 359             }
 360
 361             throwIfCancelled();
 362
 363             profile.setAndStoreAccountAndTransactionListFromWeb(list, transactions);
 364
 365             return null;
 366         }
 367     }
 368     private @NonNull
 369     LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
 370                                       ArrayList<LedgerAccount> createdAccounts) {
 371         LedgerAccount acc = map.get(accountName);
 372
 373         if (acc != null)
 374             return acc;
 375
 376         String parentName = LedgerAccount.extractParentName(accountName);
 377         LedgerAccount parentAccount;
 378         if (parentName != null) {
 379             parentAccount = ensureAccountExists(parentName, map, createdAccounts);
 380         }
 381         else {
 382             parentAccount = null;
 383         }
 384
 385         acc = new LedgerAccount(profile, accountName, parentAccount);
 386         createdAccounts.add(acc);
 387         return acc;
 388     }
 389     private boolean retrieveAccountList() throws IOException, HTTPException {
 390         Progress progress = new Progress();
 391
 392         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
 393         http.setAllowUserInteraction(false);
 394         switch (http.getResponseCode()) {
 395             case 200:
 396                 break;
 397             case 404:
 398                 return false;
 399             default:
 400                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 401         }
 402         publishProgress(progress);
 403         SQLiteDatabase db = App.getDatabase();
 404         ArrayList<LedgerAccount> list = new ArrayList<>();
 405         HashMap<String, LedgerAccount> map = new HashMap<>();
 406         HashMap<String, LedgerAccount> currentMap = new HashMap<>();
 407         for (LedgerAccount acc : Objects.requireNonNull(profile.getAllAccounts()))
 408             currentMap.put(acc.getName(), acc);
 409         try (InputStream resp = http.getInputStream()) {
 410             if (http.getResponseCode() != 200)
 411                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 412
 413             AccountListParser parser = new AccountListParser(resp);
 414
 415             while (true) {
 416                 throwIfCancelled();
 417                 ParsedLedgerAccount parsedAccount = parser.nextAccount();
 418                 if (parsedAccount == null) {
 419                     break;
 420                 }
 421
 422                 final String accName = parsedAccount.getAname();
 423                 LedgerAccount acc = map.get(accName);
 424                 if (acc != null)
 425                     throw new RuntimeException(
 426                             String.format("Account '%s' already present", acc.getName()));
 427                 String parentName = LedgerAccount.extractParentName(accName);
 428                 ArrayList<LedgerAccount> createdParents = new ArrayList<>();
 429                 LedgerAccount parent;
 430                 if (parentName == null) {
 431                     parent = null;
 432                 }
 433                 else {
 434                     parent = ensureAccountExists(parentName, map, createdParents);
 435                     parent.setHasSubAccounts(true);
 436                 }
 437                 acc = new LedgerAccount(profile, accName, parent);
 438                 list.add(acc);
 439                 map.put(accName, acc);
 440
 441                 String lastCurrency = null;
 442                 float lastCurrencyAmount = 0;
 443                 for (ParsedBalance b : parsedAccount.getAibalance()) {
 444                     throwIfCancelled();
 445                     final String currency = b.getAcommodity();
 446                     final float amount = b.getAquantity()
 447                                           .asFloat();
 448                     if (currency.equals(lastCurrency)) {
 449                         lastCurrencyAmount += amount;
 450                     }
 451                     else {
 452                         if (lastCurrency != null) {
 453                             acc.addAmount(lastCurrencyAmount, lastCurrency);
 454                         }
 455                         lastCurrency = currency;
 456                         lastCurrencyAmount = amount;
 457                     }
 458                 }
 459                 if (lastCurrency != null) {
 460                     acc.addAmount(lastCurrencyAmount, lastCurrency);
 461                 }
 462                 for (LedgerAccount p : createdParents)
 463                     acc.propagateAmountsTo(p);
 464             }
 465             throwIfCancelled();
 466         }
 467
 468         // the current account tree may have changed, update the new-to be tree to match
 469         for (LedgerAccount acc : list) {
 470             LedgerAccount prevData = currentMap.get(acc.getName());
 471             if (prevData != null) {
 472                 acc.setExpanded(prevData.isExpanded());
 473                 acc.setAmountsExpanded(prevData.amountsExpanded());
 474             }
 475         }
 476
 477         profile.setAndStoreAccountListFromWeb(list);
 478         return true;
 479     }
 480     private boolean retrieveTransactionList() throws IOException, ParseException, HTTPException {
 481         Progress progress = new Progress();
 482         int maxTransactionId = Data.transactions.size();
 483
 484         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
 485         http.setAllowUserInteraction(false);
 486         publishProgress(progress);
 487         switch (http.getResponseCode()) {
 488             case 200:
 489                 break;
 490             case 404:
 491                 return false;
 492             default:
 493                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 494         }
 495         try (InputStream resp = http.getInputStream()) {
 496             throwIfCancelled();
 497             ArrayList<LedgerTransaction> trList = new ArrayList<>();
 498
 499             TransactionListParser parser = new TransactionListParser(resp);
 500
 501             int processedTransactionCount = 0;
 502
 503             while (true) {
 504                 throwIfCancelled();
 505                 ParsedLedgerTransaction parsedTransaction = parser.nextTransaction();
 506                 throwIfCancelled();
 507                 if (parsedTransaction == null)
 508                     break;
 509
 510                 LedgerTransaction transaction = parsedTransaction.asLedgerTransaction();
 511                 trList.add(transaction);
 512
 513                 progress.setProgress(++processedTransactionCount);
 514                 publishProgress(progress);
 515             }
 516
 517             throwIfCancelled();
 518             profile.setAndStoreTransactionList(trList);
 519         }
 520
 521         return true;
 522     }
 523
 524     @SuppressLint("DefaultLocale")
 525     @Override
 526     protected String doInBackground(Void... params) {
 527         Data.backgroundTaskStarted();
 528         try {
 529             if (!retrieveAccountList() || !retrieveTransactionList())
 530                 return retrieveTransactionListLegacy();
 531             return null;
 532         }
 533         catch (MalformedURLException e) {
 534             e.printStackTrace();
 535             return "Invalid server URL";
 536         }
 537         catch (HTTPException e) {
 538             e.printStackTrace();
 539             return String.format("HTTP error %d: %s", e.getResponseCode(), e.getResponseMessage());
 540         }
 541         catch (IOException e) {
 542             e.printStackTrace();
 543             return e.getLocalizedMessage();
 544         }
 545         catch (ParseException e) {
 546             e.printStackTrace();
 547             return "Network error";
 548         }
 549         catch (OperationCanceledException e) {
 550             e.printStackTrace();
 551             return "Operation cancelled";
 552         }
 553         finally {
 554             Data.backgroundTaskFinished();
 555         }
 556     }
 557     private MainActivity getContext() {
 558         return contextRef.get();
 559     }
 560     private void throwIfCancelled() {
 561         if (isCancelled())
 562             throw new OperationCanceledException(null);
 563     }
 564     private enum ParserState {
 565         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
 566         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
 567     }
 568
 569     public static class Progress {
 570         public static final int INDETERMINATE = -1;
 571         private int progress;
 572         private int total;
 573         Progress() {
 574             this(INDETERMINATE, INDETERMINATE);
 575         }
 576         Progress(int progress, int total) {
 577             this.progress = progress;
 578             this.total = total;
 579         }
 580         public int getProgress() {
 581             return progress;
 582         }
 583         protected void setProgress(int progress) {
 584             this.progress = progress;
 585         }
 586         public int getTotal() {
 587             return total;
 588         }
 589         protected void setTotal(int total) {
 590             this.total = total;
 591         }
 592     }
 593
 594     private static class TransactionParserException extends IllegalStateException {
 595         TransactionParserException(String message) {
 596             super(message);
 597         }
 598     }
 599 }