app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java

   1 /*
   2  * Copyright © 2021 Damyan Ivanov.
   3  * This file is part of MoLe.
   4  * MoLe is free software: you can distribute it and/or modify it
   5  * under the term of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your opinion), any later version.
   8  *
   9  * MoLe is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12  * GNU General Public License terms for details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
  16  */
  17
  18 package net.ktnx.mobileledger.async;
  19
  20 import android.annotation.SuppressLint;
  21 import android.os.AsyncTask;
  22 import android.os.OperationCanceledException;
  23
  24 import androidx.annotation.NonNull;
  25 import androidx.room.Transaction;
  26
  27 import com.fasterxml.jackson.core.JsonParseException;
  28 import com.fasterxml.jackson.databind.RuntimeJsonMappingException;
  29
  30 import net.ktnx.mobileledger.dao.AccountDAO;
  31 import net.ktnx.mobileledger.dao.AccountValueDAO;
  32 import net.ktnx.mobileledger.dao.TransactionAccountDAO;
  33 import net.ktnx.mobileledger.dao.TransactionDAO;
  34 import net.ktnx.mobileledger.db.Account;
  35 import net.ktnx.mobileledger.db.AccountWithAmounts;
  36 import net.ktnx.mobileledger.db.DB;
  37 import net.ktnx.mobileledger.db.Option;
  38 import net.ktnx.mobileledger.db.Profile;
  39 import net.ktnx.mobileledger.db.TransactionAccount;
  40 import net.ktnx.mobileledger.db.TransactionWithAccounts;
  41 import net.ktnx.mobileledger.err.HTTPException;
  42 import net.ktnx.mobileledger.json.API;
  43 import net.ktnx.mobileledger.json.AccountListParser;
  44 import net.ktnx.mobileledger.json.ApiNotSupportedException;
  45 import net.ktnx.mobileledger.json.TransactionListParser;
  46 import net.ktnx.mobileledger.model.Data;
  47 import net.ktnx.mobileledger.model.LedgerAccount;
  48 import net.ktnx.mobileledger.model.LedgerTransaction;
  49 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
  50 import net.ktnx.mobileledger.ui.MainModel;
  51 import net.ktnx.mobileledger.utils.Logger;
  52 import net.ktnx.mobileledger.utils.NetworkUtil;
  53 import net.ktnx.mobileledger.utils.Profiler;
  54
  55 import java.io.BufferedReader;
  56 import java.io.IOException;
  57 import java.io.InputStream;
  58 import java.io.InputStreamReader;
  59 import java.net.HttpURLConnection;
  60 import java.net.MalformedURLException;
  61 import java.net.URLDecoder;
  62 import java.nio.charset.StandardCharsets;
  63 import java.text.ParseException;
  64 import java.util.ArrayList;
  65 import java.util.Collections;
  66 import java.util.Date;
  67 import java.util.HashMap;
  68 import java.util.List;
  69 import java.util.Locale;
  70 import java.util.Objects;
  71 import java.util.regex.Matcher;
  72 import java.util.regex.Pattern;
  73
  74
  75 public class RetrieveTransactionsTask extends
  76         AsyncTask<Void, RetrieveTransactionsTask.Progress, RetrieveTransactionsTask.Result> {
  77     private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
  78     private static final Pattern reComment = Pattern.compile("^\\s*;");
  79     private static final Pattern reTransactionStart = Pattern.compile(
  80             "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
  81             "\"[^\"]*>([\\d.-]+)</td>");
  82     private static final Pattern reTransactionDescription =
  83             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
  84     private static final Pattern reTransactionDetails = Pattern.compile(
  85             "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
  86             "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
  87     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
  88     private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
  89     private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
  90     private static final String TAG = "RTT";
  91     // %3A is '='
  92     private final Pattern reAccountName =
  93             Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
  94     private final Pattern reAccountValue = Pattern.compile(
  95             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
  96     private final MainModel mainModel;
  97     private final Profile profile;
  98     private int expectedPostingsCount = -1;
  99     public RetrieveTransactionsTask(@NonNull MainModel mainModel, @NonNull Profile profile) {
 100         this.mainModel = mainModel;
 101         this.profile = profile;
 102     }
 103     private static void L(String msg) {
 104         //debug("transaction-parser", msg);
 105     }
 106     static LedgerTransactionAccount parseTransactionAccountLine(String line) {
 107         Matcher m = reTransactionDetails.matcher(line);
 108         if (m.find()) {
 109             String postingStatus = m.group(1);
 110             String acc_name = m.group(2);
 111             String currencyPre = m.group(3);
 112             String amount = Objects.requireNonNull(m.group(4));
 113             String currencyPost = m.group(5);
 114
 115             String currency = null;
 116             if ((currencyPre != null) && (currencyPre.length() > 0)) {
 117                 if ((currencyPost != null) && (currencyPost.length() > 0))
 118                     return null;
 119                 currency = currencyPre;
 120             }
 121             else if ((currencyPost != null) && (currencyPost.length() > 0)) {
 122                 currency = currencyPost;
 123             }
 124
 125             amount = amount.replace(',', '.');
 126
 127             return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
 128         }
 129         else {
 130             return null;
 131         }
 132     }
 133     @Override
 134     protected void onProgressUpdate(Progress... values) {
 135         super.onProgressUpdate(values);
 136         Data.backgroundTaskProgress.postValue(values[0]);
 137     }
 138     @Override
 139     protected void onPostExecute(Result result) {
 140         super.onPostExecute(result);
 141         Progress progress = new Progress();
 142         progress.setState(ProgressState.FINISHED);
 143         progress.setError(result.error);
 144         onProgressUpdate(progress);
 145     }
 146     @Override
 147     protected void onCancelled() {
 148         super.onCancelled();
 149         Progress progress = new Progress();
 150         progress.setState(ProgressState.FINISHED);
 151         onProgressUpdate(progress);
 152     }
 153     private void retrieveTransactionListLegacy(List<LedgerAccount> accounts,
 154                                                List<LedgerTransaction> transactions)
 155             throws IOException, HTTPException {
 156         Progress progress = Progress.indeterminate();
 157         progress.setState(ProgressState.RUNNING);
 158         progress.setTotal(expectedPostingsCount);
 159         int maxTransactionId = -1;
 160         HashMap<String, LedgerAccount> map = new HashMap<>();
 161         LedgerAccount lastAccount = null;
 162         ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
 163
 164         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
 165         http.setAllowUserInteraction(false);
 166         publishProgress(progress);
 167         if (http.getResponseCode() != 200)
 168             throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 169
 170         try (InputStream resp = http.getInputStream()) {
 171             if (http.getResponseCode() != 200)
 172                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 173
 174             int matchedTransactionsCount = 0;
 175
 176             ParserState state = ParserState.EXPECTING_ACCOUNT;
 177             String line;
 178             BufferedReader buf =
 179                     new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
 180
 181             int processedTransactionCount = 0;
 182             int transactionId = 0;
 183             LedgerTransaction transaction = null;
 184             LINES:
 185             while ((line = buf.readLine()) != null) {
 186                 throwIfCancelled();
 187                 Matcher m;
 188                 m = reComment.matcher(line);
 189                 if (m.find()) {
 190                     // TODO: comments are ignored for now
 191 //                            Log.v("transaction-parser", "Ignoring comment");
 192                     continue;
 193                 }
 194                 //L(String.format("State is %d", updating));
 195                 switch (state) {
 196                     case EXPECTING_ACCOUNT:
 197                         if (line.equals("<h2>General Journal</h2>")) {
 198                             state = ParserState.EXPECTING_TRANSACTION;
 199                             L("→ expecting transaction");
 200                             continue;
 201                         }
 202                         m = reAccountName.matcher(line);
 203                         if (m.find()) {
 204                             String acct_encoded = m.group(1);
 205                             String accName = URLDecoder.decode(acct_encoded, "UTF-8");
 206                             accName = accName.replace("\"", "");
 207                             L(String.format("found account: %s", accName));
 208
 209                             lastAccount = map.get(accName);
 210                             if (lastAccount != null) {
 211                                 L(String.format("ignoring duplicate account '%s'", accName));
 212                                 continue;
 213                             }
 214                             String parentAccountName = LedgerAccount.extractParentName(accName);
 215                             LedgerAccount parentAccount;
 216                             if (parentAccountName != null) {
 217                                 parentAccount = ensureAccountExists(parentAccountName, map,
 218                                         syntheticAccounts);
 219                             }
 220                             else {
 221                                 parentAccount = null;
 222                             }
 223                             lastAccount = new LedgerAccount(accName, parentAccount);
 224
 225                             accounts.add(lastAccount);
 226                             map.put(accName, lastAccount);
 227
 228                             state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
 229                             L("→ expecting account amount");
 230                         }
 231                         break;
 232
 233                     case EXPECTING_ACCOUNT_AMOUNT:
 234                         m = reAccountValue.matcher(line);
 235                         boolean match_found = false;
 236                         while (m.find()) {
 237                             throwIfCancelled();
 238
 239                             match_found = true;
 240                             String value = Objects.requireNonNull(m.group(1));
 241                             String currency = m.group(2);
 242                             if (currency == null)
 243                                 currency = "";
 244
 245                             {
 246                                 Matcher tmpM = reDecimalComma.matcher(value);
 247                                 if (tmpM.find()) {
 248                                     value = value.replace(".", "");
 249                                     value = value.replace(',', '.');
 250                                 }
 251
 252                                 tmpM = reDecimalPoint.matcher(value);
 253                                 if (tmpM.find()) {
 254                                     value = value.replace(",", "");
 255                                     value = value.replace(" ", "");
 256                                 }
 257                             }
 258                             L("curr=" + currency + ", value=" + value);
 259                             final float val = Float.parseFloat(value);
 260                             lastAccount.addAmount(val, currency);
 261                             for (LedgerAccount syn : syntheticAccounts) {
 262                                 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
 263                                         currency, val, syn.getName()));
 264                                 syn.addAmount(val, currency);
 265                             }
 266                         }
 267
 268                         if (match_found) {
 269                             syntheticAccounts.clear();
 270                             state = ParserState.EXPECTING_ACCOUNT;
 271                             L("→ expecting account");
 272                         }
 273
 274                         break;
 275
 276                     case EXPECTING_TRANSACTION:
 277                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 278                             continue;
 279                         m = reTransactionStart.matcher(line);
 280                         if (m.find()) {
 281                             transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
 282                             state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
 283                             L(String.format(Locale.ENGLISH,
 284                                     "found transaction %d → expecting description", transactionId));
 285                             progress.setProgress(++processedTransactionCount);
 286                             if (maxTransactionId < transactionId)
 287                                 maxTransactionId = transactionId;
 288                             if ((progress.isIndeterminate()) ||
 289                                 (progress.getTotal() < transactionId))
 290                                 progress.setTotal(transactionId);
 291                             publishProgress(progress);
 292                         }
 293                         m = reEnd.matcher(line);
 294                         if (m.find()) {
 295                             L("--- transaction value complete ---");
 296                             break LINES;
 297                         }
 298                         break;
 299
 300                     case EXPECTING_TRANSACTION_DESCRIPTION:
 301                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 302                             continue;
 303                         m = reTransactionDescription.matcher(line);
 304                         if (m.find()) {
 305                             if (transactionId == 0)
 306                                 throw new TransactionParserException(
 307                                         "Transaction Id is 0 while expecting description");
 308
 309                             String date = Objects.requireNonNull(m.group(1));
 310                             try {
 311                                 int equalsIndex = date.indexOf('=');
 312                                 if (equalsIndex >= 0)
 313                                     date = date.substring(equalsIndex + 1);
 314                                 transaction =
 315                                         new LedgerTransaction(transactionId, date, m.group(2));
 316                             }
 317                             catch (ParseException e) {
 318                                 throw new TransactionParserException(
 319                                         String.format("Error parsing date '%s'", date));
 320                             }
 321                             state = ParserState.EXPECTING_TRANSACTION_DETAILS;
 322                             L(String.format(Locale.ENGLISH,
 323                                     "transaction %d created for %s (%s) →" + " expecting details",
 324                                     transactionId, date, m.group(2)));
 325                         }
 326                         break;
 327
 328                     case EXPECTING_TRANSACTION_DETAILS:
 329                         if (line.isEmpty()) {
 330                             // transaction data collected
 331
 332                             transaction.finishLoading();
 333                             transactions.add(transaction);
 334
 335                             state = ParserState.EXPECTING_TRANSACTION;
 336                             L(String.format("transaction %s parsed → expecting transaction",
 337                                     transaction.getLedgerId()));
 338
 339 // sounds like a good idea, but transaction-1 may not be the first one chronologically
 340 // for example, when you add the initial seeding transaction after entering some others
 341 //                                            if (transactionId == 1) {
 342 //                                                L("This was the initial transaction.
 343 //                                                Terminating " +
 344 //                                                  "parser");
 345 //                                                break LINES;
 346 //                                            }
 347                         }
 348                         else {
 349                             LedgerTransactionAccount lta = parseTransactionAccountLine(line);
 350                             if (lta != null) {
 351                                 transaction.addAccount(lta);
 352                                 L(String.format(Locale.ENGLISH, "%d: %s = %s",
 353                                         transaction.getLedgerId(), lta.getAccountName(),
 354                                         lta.getAmount()));
 355                             }
 356                             else
 357                                 throw new IllegalStateException(
 358                                         String.format("Can't parse transaction %d details: %s",
 359                                                 transactionId, line));
 360                         }
 361                         break;
 362                     default:
 363                         throw new RuntimeException(
 364                                 String.format("Unknown parser updating %s", state.name()));
 365                 }
 366             }
 367
 368             throwIfCancelled();
 369         }
 370     }
 371     @NonNull
 372     public LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
 373                                              ArrayList<LedgerAccount> createdAccounts) {
 374         LedgerAccount acc = map.get(accountName);
 375
 376         if (acc != null)
 377             return acc;
 378
 379         String parentName = LedgerAccount.extractParentName(accountName);
 380         LedgerAccount parentAccount;
 381         if (parentName != null) {
 382             parentAccount = ensureAccountExists(parentName, map, createdAccounts);
 383         }
 384         else {
 385             parentAccount = null;
 386         }
 387
 388         acc = new LedgerAccount(accountName, parentAccount);
 389         createdAccounts.add(acc);
 390         return acc;
 391     }
 392     public void addNumberOfPostings(int number) {
 393         expectedPostingsCount += number;
 394     }
 395     private List<LedgerAccount> retrieveAccountList()
 396             throws IOException, HTTPException, ApiNotSupportedException {
 397         final API apiVersion = API.valueOf(profile.getApiVersion());
 398         if (apiVersion.equals(API.auto)) {
 399             return retrieveAccountListAnyVersion();
 400         }
 401         else if (apiVersion.equals(API.html)) {
 402             Logger.debug("json",
 403                     "Declining using JSON API for /accounts with configured legacy API version");
 404             return null;
 405         }
 406         else {
 407             return retrieveAccountListForVersion(apiVersion);
 408         }
 409     }
 410     private List<LedgerAccount> retrieveAccountListAnyVersion()
 411             throws ApiNotSupportedException, IOException, HTTPException {
 412         for (API ver : API.allVersions) {
 413             try {
 414                 return retrieveAccountListForVersion(ver);
 415             }
 416             catch (JsonParseException | RuntimeJsonMappingException e) {
 417                 Logger.debug("json",
 418                         String.format(Locale.US, "Error during account list retrieval using API %s",
 419                                 ver.getDescription()), e);
 420             }
 421
 422         }
 423
 424         throw new ApiNotSupportedException();
 425     }
 426     private List<LedgerAccount> retrieveAccountListForVersion(API version)
 427             throws IOException, HTTPException {
 428         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
 429         http.setAllowUserInteraction(false);
 430         switch (http.getResponseCode()) {
 431             case 200:
 432                 break;
 433             case 404:
 434                 return null;
 435             default:
 436                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 437         }
 438         publishProgress(Progress.indeterminate());
 439         ArrayList<LedgerAccount> list = new ArrayList<>();
 440         HashMap<String, LedgerAccount> map = new HashMap<>();
 441         throwIfCancelled();
 442         try (InputStream resp = http.getInputStream()) {
 443             throwIfCancelled();
 444             if (http.getResponseCode() != 200)
 445                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 446
 447             AccountListParser parser = AccountListParser.forApiVersion(version, resp);
 448             expectedPostingsCount = 0;
 449
 450             while (true) {
 451                 throwIfCancelled();
 452                 LedgerAccount acc = parser.nextAccount(this, map);
 453                 if (acc == null)
 454                     break;
 455                 list.add(acc);
 456             }
 457             throwIfCancelled();
 458         }
 459
 460         return list;
 461     }
 462     private List<LedgerTransaction> retrieveTransactionList()
 463             throws ParseException, HTTPException, IOException, ApiNotSupportedException {
 464         final API apiVersion = API.valueOf(profile.getApiVersion());
 465         if (apiVersion.equals(API.auto)) {
 466             return retrieveTransactionListAnyVersion();
 467         }
 468         else if (apiVersion.equals(API.html)) {
 469             Logger.debug("json",
 470                     "Declining using JSON API for /accounts with configured legacy API version");
 471             return null;
 472         }
 473         else {
 474             return retrieveTransactionListForVersion(apiVersion);
 475         }
 476
 477     }
 478     private List<LedgerTransaction> retrieveTransactionListAnyVersion()
 479             throws ApiNotSupportedException {
 480         for (API ver : API.allVersions) {
 481             try {
 482                 return retrieveTransactionListForVersion(ver);
 483             }
 484             catch (Exception e) {
 485                 Logger.debug("json",
 486                         String.format(Locale.US, "Error during account list retrieval using API %s",
 487                                 ver.getDescription()));
 488             }
 489
 490         }
 491
 492         throw new ApiNotSupportedException();
 493     }
 494     private List<LedgerTransaction> retrieveTransactionListForVersion(API apiVersion)
 495             throws IOException, ParseException, HTTPException {
 496         Progress progress = new Progress();
 497         progress.setTotal(expectedPostingsCount);
 498
 499         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
 500         http.setAllowUserInteraction(false);
 501         publishProgress(progress);
 502         switch (http.getResponseCode()) {
 503             case 200:
 504                 break;
 505             case 404:
 506                 return null;
 507             default:
 508                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 509         }
 510         ArrayList<LedgerTransaction> trList = new ArrayList<>();
 511         try (InputStream resp = http.getInputStream()) {
 512             throwIfCancelled();
 513
 514             TransactionListParser parser = TransactionListParser.forApiVersion(apiVersion, resp);
 515
 516             int processedPostings = 0;
 517
 518             while (true) {
 519                 throwIfCancelled();
 520                 LedgerTransaction transaction = parser.nextTransaction();
 521                 throwIfCancelled();
 522                 if (transaction == null)
 523                     break;
 524
 525                 trList.add(transaction);
 526
 527                 progress.setProgress(processedPostings += transaction.getAccounts()
 528                                                                      .size());
 529 //                Logger.debug("trParser",
 530 //                        String.format(Locale.US, "Parsed transaction %d - %s", transaction
 531 //                        .getId(),
 532 //                                transaction.getDescription()));
 533 //                for (LedgerTransactionAccount acc : transaction.getAccounts()) {
 534 //                    Logger.debug("trParser",
 535 //                            String.format(Locale.US, "  %s", acc.getAccountName()));
 536 //                }
 537                 publishProgress(progress);
 538             }
 539
 540             throwIfCancelled();
 541         }
 542
 543         // json interface returns transactions if file order and the rest of the machinery
 544         // expects them in reverse chronological order
 545         Collections.sort(trList, (o1, o2) -> {
 546             int res = o2.getDate()
 547                         .compareTo(o1.getDate());
 548             if (res != 0)
 549                 return res;
 550             return Long.compare(o2.getLedgerId(), o1.getLedgerId());
 551         });
 552         return trList;
 553     }
 554
 555     @SuppressLint("DefaultLocale")
 556     @Override
 557     protected Result doInBackground(Void... params) {
 558         Data.backgroundTaskStarted();
 559         List<LedgerAccount> accounts;
 560         List<LedgerTransaction> transactions;
 561         try {
 562             accounts = retrieveAccountList();
 563             // accounts is null in API-version auto-detection and means
 564             // requesting 'html' API version via the JSON classes
 565             // this can't work, and the null results in the legacy code below
 566             // being called
 567             if (accounts == null)
 568                 transactions = null;
 569             else
 570                 transactions = retrieveTransactionList();
 571
 572             if (accounts == null || transactions == null) {
 573                 accounts = new ArrayList<>();
 574                 transactions = new ArrayList<>();
 575                 retrieveTransactionListLegacy(accounts, transactions);
 576             }
 577
 578             storeAccountsAndTransactions(accounts, transactions);
 579
 580             mainModel.updateDisplayedTransactionsFromWeb(transactions);
 581
 582             return new Result(accounts, transactions);
 583         }
 584         catch (MalformedURLException e) {
 585             e.printStackTrace();
 586             return new Result("Invalid server URL");
 587         }
 588         catch (HTTPException e) {
 589             e.printStackTrace();
 590             return new Result(
 591                     String.format("HTTP error %d: %s", e.getResponseCode(), e.getMessage()));
 592         }
 593         catch (IOException e) {
 594             e.printStackTrace();
 595             return new Result(e.getLocalizedMessage());
 596         }
 597         catch (RuntimeJsonMappingException e) {
 598             e.printStackTrace();
 599             return new Result(Result.ERR_JSON_PARSER_ERROR);
 600         }
 601         catch (ParseException e) {
 602             e.printStackTrace();
 603             return new Result("Network error");
 604         }
 605         catch (OperationCanceledException e) {
 606             e.printStackTrace();
 607             return new Result("Operation cancelled");
 608         }
 609         catch (ApiNotSupportedException e) {
 610             e.printStackTrace();
 611             return new Result("Server version not supported");
 612         }
 613         finally {
 614             Data.backgroundTaskFinished();
 615         }
 616     }
 617     @Transaction
 618     private void storeAccountsAndTransactions(List<LedgerAccount> accounts,
 619                                               List<LedgerTransaction> transactions) {
 620         AccountDAO accDao = DB.get()
 621                               .getAccountDAO();
 622         TransactionDAO trDao = DB.get()
 623                                  .getTransactionDAO();
 624         TransactionAccountDAO trAccDao = DB.get()
 625                                            .getTransactionAccountDAO();
 626         AccountValueDAO valDao = DB.get()
 627                                    .getAccountValueDAO();
 628
 629         Logger.debug(TAG, "Preparing account list");
 630         final List<AccountWithAmounts> list = new ArrayList<>();
 631         for (LedgerAccount acc : accounts) {
 632             final AccountWithAmounts a = acc.toDBOWithAmounts();
 633             Account existing = accDao.getByNameSync(profile.getId(), acc.getName());
 634             if (existing != null) {
 635                 a.account.setExpanded(existing.isExpanded());
 636                 a.account.setAmountsExpanded(existing.isAmountsExpanded());
 637                 a.account.setId(
 638                         existing.getId()); // not strictly needed, but since we have it anyway...
 639             }
 640
 641             list.add(a);
 642         }
 643         Logger.debug(TAG, "Account list prepared. Storing");
 644         accDao.storeAccountsSync(list, profile.getId());
 645         Logger.debug(TAG, "Account list stored");
 646
 647         Profiler tranProfiler = new Profiler("transactions");
 648         Profiler tranAccProfiler = new Profiler("transaction accounts");
 649
 650         Logger.debug(TAG, "Storing transactions");
 651         long trGen = trDao.getGenerationSync(profile.getId());
 652         for (LedgerTransaction tr : transactions) {
 653             TransactionWithAccounts tran = tr.toDBO();
 654             tran.transaction.setGeneration(trGen);
 655             tran.transaction.setProfileId(profile.getId());
 656
 657             tranProfiler.opStart();
 658             tran.transaction.setId(trDao.insertSync(tran.transaction));
 659             tranProfiler.opEnd();
 660
 661             for (TransactionAccount trAcc : tran.accounts) {
 662                 trAcc.setGeneration(trGen);
 663                 trAcc.setTransactionId(tran.transaction.getId());
 664                 tranAccProfiler.opStart();
 665                 trAcc.setId(trAccDao.insertSync(trAcc));
 666                 tranAccProfiler.opEnd();
 667             }
 668         }
 669
 670         tranProfiler.dumpStats();
 671         tranAccProfiler.dumpStats();
 672
 673         Logger.debug(TAG, "Transactions stored. Purging old");
 674         trDao.purgeOldTransactionsSync(profile.getId(), trGen);
 675         Logger.debug(TAG, "Old transactions purged");
 676
 677         DB.get()
 678           .getOptionDAO()
 679           .insertSync(new Option(profile.getId(), Option.OPT_LAST_SCRAPE,
 680                   String.valueOf((new Date()).getTime())));
 681     }
 682     public void throwIfCancelled() {
 683         if (isCancelled())
 684             throw new OperationCanceledException(null);
 685     }
 686     private enum ParserState {
 687         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
 688         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
 689     }
 690
 691     public enum ProgressState {STARTING, RUNNING, FINISHED}
 692
 693     public static class Progress {
 694         private int progress;
 695         private int total;
 696         private ProgressState state = ProgressState.RUNNING;
 697         private String error = null;
 698         private boolean indeterminate;
 699         Progress() {
 700             indeterminate = true;
 701         }
 702         Progress(int progress, int total) {
 703             this.indeterminate = false;
 704             this.progress = progress;
 705             this.total = total;
 706         }
 707         public static Progress indeterminate() {
 708             return new Progress();
 709         }
 710         public static Progress finished(String error) {
 711             Progress p = new Progress();
 712             p.setState(ProgressState.FINISHED);
 713             p.setError(error);
 714             return p;
 715         }
 716         public int getProgress() {
 717             ensureState(ProgressState.RUNNING);
 718             return progress;
 719         }
 720         protected void setProgress(int progress) {
 721             this.progress = progress;
 722             this.state = ProgressState.RUNNING;
 723         }
 724         public int getTotal() {
 725             ensureState(ProgressState.RUNNING);
 726             return total;
 727         }
 728         protected void setTotal(int total) {
 729             this.total = total;
 730             state = ProgressState.RUNNING;
 731             indeterminate = total == -1;
 732         }
 733         private void ensureState(ProgressState wanted) {
 734             if (state != wanted)
 735                 throw new IllegalStateException(
 736                         String.format("Bad state: %s, expected %s", state, wanted));
 737         }
 738         public ProgressState getState() {
 739             return state;
 740         }
 741         public void setState(ProgressState state) {
 742             this.state = state;
 743         }
 744         public String getError() {
 745             ensureState(ProgressState.FINISHED);
 746             return error;
 747         }
 748         public void setError(String error) {
 749             this.error = error;
 750             state = ProgressState.FINISHED;
 751         }
 752         public boolean isIndeterminate() {
 753             return indeterminate;
 754         }
 755         public void setIndeterminate(boolean indeterminate) {
 756             this.indeterminate = indeterminate;
 757         }
 758     }
 759
 760     private static class TransactionParserException extends IllegalStateException {
 761         TransactionParserException(String message) {
 762             super(message);
 763         }
 764     }
 765
 766     public static class Result {
 767         public static String ERR_JSON_PARSER_ERROR = "err_json_parser";
 768         public String error;
 769         public List<LedgerAccount> accounts;
 770         public List<LedgerTransaction> transactions;
 771         Result(String error) {
 772             this.error = error;
 773         }
 774         Result(List<LedgerAccount> accounts, List<LedgerTransaction> transactions) {
 775             this.accounts = accounts;
 776             this.transactions = transactions;
 777         }
 778     }
 779 }