app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java

   1 /*
   2  * Copyright © 2021 Damyan Ivanov.
   3  * This file is part of MoLe.
   4  * MoLe is free software: you can distribute it and/or modify it
   5  * under the term of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your opinion), any later version.
   8  *
   9  * MoLe is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12  * GNU General Public License terms for details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
  16  */
  17
  18 package net.ktnx.mobileledger.async;
  19
  20 import android.annotation.SuppressLint;
  21 import android.database.sqlite.SQLiteDatabase;
  22 import android.os.AsyncTask;
  23 import android.os.OperationCanceledException;
  24
  25 import androidx.annotation.NonNull;
  26 import androidx.room.Transaction;
  27
  28 import com.fasterxml.jackson.core.JsonParseException;
  29 import com.fasterxml.jackson.databind.RuntimeJsonMappingException;
  30
  31 import net.ktnx.mobileledger.App;
  32 import net.ktnx.mobileledger.dao.AccountDAO;
  33 import net.ktnx.mobileledger.dao.AccountValueDAO;
  34 import net.ktnx.mobileledger.dao.TransactionAccountDAO;
  35 import net.ktnx.mobileledger.dao.TransactionDAO;
  36 import net.ktnx.mobileledger.db.Account;
  37 import net.ktnx.mobileledger.db.AccountWithAmounts;
  38 import net.ktnx.mobileledger.db.DB;
  39 import net.ktnx.mobileledger.db.Option;
  40 import net.ktnx.mobileledger.db.Profile;
  41 import net.ktnx.mobileledger.db.TransactionAccount;
  42 import net.ktnx.mobileledger.db.TransactionWithAccounts;
  43 import net.ktnx.mobileledger.err.HTTPException;
  44 import net.ktnx.mobileledger.json.API;
  45 import net.ktnx.mobileledger.json.AccountListParser;
  46 import net.ktnx.mobileledger.json.ApiNotSupportedException;
  47 import net.ktnx.mobileledger.json.TransactionListParser;
  48 import net.ktnx.mobileledger.model.Data;
  49 import net.ktnx.mobileledger.model.LedgerAccount;
  50 import net.ktnx.mobileledger.model.LedgerTransaction;
  51 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
  52 import net.ktnx.mobileledger.ui.MainModel;
  53 import net.ktnx.mobileledger.utils.Logger;
  54 import net.ktnx.mobileledger.utils.MLDB;
  55 import net.ktnx.mobileledger.utils.NetworkUtil;
  56
  57 import java.io.BufferedReader;
  58 import java.io.IOException;
  59 import java.io.InputStream;
  60 import java.io.InputStreamReader;
  61 import java.net.HttpURLConnection;
  62 import java.net.MalformedURLException;
  63 import java.net.URLDecoder;
  64 import java.nio.charset.StandardCharsets;
  65 import java.text.ParseException;
  66 import java.util.ArrayList;
  67 import java.util.Collections;
  68 import java.util.Date;
  69 import java.util.HashMap;
  70 import java.util.List;
  71 import java.util.Locale;
  72 import java.util.Objects;
  73 import java.util.regex.Matcher;
  74 import java.util.regex.Pattern;
  75
  76
  77 public class RetrieveTransactionsTask extends
  78         AsyncTask<Void, RetrieveTransactionsTask.Progress, RetrieveTransactionsTask.Result> {
  79     private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
  80     private static final Pattern reComment = Pattern.compile("^\\s*;");
  81     private static final Pattern reTransactionStart = Pattern.compile(
  82             "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
  83             "\"[^\"]*>([\\d.-]+)</td>");
  84     private static final Pattern reTransactionDescription =
  85             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
  86     private static final Pattern reTransactionDetails = Pattern.compile(
  87             "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
  88             "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
  89     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
  90     private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
  91     private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
  92     // %3A is '='
  93     private final Pattern reAccountName =
  94             Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
  95     private final Pattern reAccountValue = Pattern.compile(
  96             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
  97     private final MainModel mainModel;
  98     private final Profile profile;
  99     private int expectedPostingsCount = -1;
 100     public RetrieveTransactionsTask(@NonNull MainModel mainModel, @NonNull Profile profile) {
 101         this.mainModel = mainModel;
 102         this.profile = profile;
 103     }
 104     private static void L(String msg) {
 105         //debug("transaction-parser", msg);
 106     }
 107     static LedgerTransactionAccount parseTransactionAccountLine(String line) {
 108         Matcher m = reTransactionDetails.matcher(line);
 109         if (m.find()) {
 110             String postingStatus = m.group(1);
 111             String acc_name = m.group(2);
 112             String currencyPre = m.group(3);
 113             String amount = Objects.requireNonNull(m.group(4));
 114             String currencyPost = m.group(5);
 115
 116             String currency = null;
 117             if ((currencyPre != null) && (currencyPre.length() > 0)) {
 118                 if ((currencyPost != null) && (currencyPost.length() > 0))
 119                     return null;
 120                 currency = currencyPre;
 121             }
 122             else if ((currencyPost != null) && (currencyPost.length() > 0)) {
 123                 currency = currencyPost;
 124             }
 125
 126             amount = amount.replace(',', '.');
 127
 128             return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
 129         }
 130         else {
 131             return null;
 132         }
 133     }
 134     @Override
 135     protected void onProgressUpdate(Progress... values) {
 136         super.onProgressUpdate(values);
 137         Data.backgroundTaskProgress.postValue(values[0]);
 138     }
 139     @Override
 140     protected void onPostExecute(Result result) {
 141         super.onPostExecute(result);
 142         Progress progress = new Progress();
 143         progress.setState(ProgressState.FINISHED);
 144         progress.setError(result.error);
 145         onProgressUpdate(progress);
 146     }
 147     @Override
 148     protected void onCancelled() {
 149         super.onCancelled();
 150         Progress progress = new Progress();
 151         progress.setState(ProgressState.FINISHED);
 152         onProgressUpdate(progress);
 153     }
 154     private void retrieveTransactionListLegacy(List<LedgerAccount> accounts,
 155                                                List<LedgerTransaction> transactions)
 156             throws IOException, HTTPException {
 157         Progress progress = Progress.indeterminate();
 158         progress.setState(ProgressState.RUNNING);
 159         progress.setTotal(expectedPostingsCount);
 160         int maxTransactionId = -1;
 161         HashMap<String, LedgerAccount> map = new HashMap<>();
 162         LedgerAccount lastAccount = null;
 163         ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
 164
 165         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
 166         http.setAllowUserInteraction(false);
 167         publishProgress(progress);
 168         if (http.getResponseCode() != 200)
 169             throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 170
 171         try (InputStream resp = http.getInputStream()) {
 172             if (http.getResponseCode() != 200)
 173                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 174
 175             int matchedTransactionsCount = 0;
 176
 177             ParserState state = ParserState.EXPECTING_ACCOUNT;
 178             String line;
 179             BufferedReader buf =
 180                     new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
 181
 182             int processedTransactionCount = 0;
 183             int transactionId = 0;
 184             LedgerTransaction transaction = null;
 185             LINES:
 186             while ((line = buf.readLine()) != null) {
 187                 throwIfCancelled();
 188                 Matcher m;
 189                 m = reComment.matcher(line);
 190                 if (m.find()) {
 191                     // TODO: comments are ignored for now
 192 //                            Log.v("transaction-parser", "Ignoring comment");
 193                     continue;
 194                 }
 195                 //L(String.format("State is %d", updating));
 196                 switch (state) {
 197                     case EXPECTING_ACCOUNT:
 198                         if (line.equals("<h2>General Journal</h2>")) {
 199                             state = ParserState.EXPECTING_TRANSACTION;
 200                             L("→ expecting transaction");
 201                             continue;
 202                         }
 203                         m = reAccountName.matcher(line);
 204                         if (m.find()) {
 205                             String acct_encoded = m.group(1);
 206                             String accName = URLDecoder.decode(acct_encoded, "UTF-8");
 207                             accName = accName.replace("\"", "");
 208                             L(String.format("found account: %s", accName));
 209
 210                             lastAccount = map.get(accName);
 211                             if (lastAccount != null) {
 212                                 L(String.format("ignoring duplicate account '%s'", accName));
 213                                 continue;
 214                             }
 215                             String parentAccountName = LedgerAccount.extractParentName(accName);
 216                             LedgerAccount parentAccount;
 217                             if (parentAccountName != null) {
 218                                 parentAccount = ensureAccountExists(parentAccountName, map,
 219                                         syntheticAccounts);
 220                             }
 221                             else {
 222                                 parentAccount = null;
 223                             }
 224                             lastAccount = new LedgerAccount(accName, parentAccount);
 225
 226                             accounts.add(lastAccount);
 227                             map.put(accName, lastAccount);
 228
 229                             state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
 230                             L("→ expecting account amount");
 231                         }
 232                         break;
 233
 234                     case EXPECTING_ACCOUNT_AMOUNT:
 235                         m = reAccountValue.matcher(line);
 236                         boolean match_found = false;
 237                         while (m.find()) {
 238                             throwIfCancelled();
 239
 240                             match_found = true;
 241                             String value = Objects.requireNonNull(m.group(1));
 242                             String currency = m.group(2);
 243                             if (currency == null)
 244                                 currency = "";
 245
 246                             {
 247                                 Matcher tmpM = reDecimalComma.matcher(value);
 248                                 if (tmpM.find()) {
 249                                     value = value.replace(".", "");
 250                                     value = value.replace(',', '.');
 251                                 }
 252
 253                                 tmpM = reDecimalPoint.matcher(value);
 254                                 if (tmpM.find()) {
 255                                     value = value.replace(",", "");
 256                                     value = value.replace(" ", "");
 257                                 }
 258                             }
 259                             L("curr=" + currency + ", value=" + value);
 260                             final float val = Float.parseFloat(value);
 261                             lastAccount.addAmount(val, currency);
 262                             for (LedgerAccount syn : syntheticAccounts) {
 263                                 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
 264                                         currency, val, syn.getName()));
 265                                 syn.addAmount(val, currency);
 266                             }
 267                         }
 268
 269                         if (match_found) {
 270                             syntheticAccounts.clear();
 271                             state = ParserState.EXPECTING_ACCOUNT;
 272                             L("→ expecting account");
 273                         }
 274
 275                         break;
 276
 277                     case EXPECTING_TRANSACTION:
 278                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 279                             continue;
 280                         m = reTransactionStart.matcher(line);
 281                         if (m.find()) {
 282                             transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
 283                             state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
 284                             L(String.format(Locale.ENGLISH,
 285                                     "found transaction %d → expecting description", transactionId));
 286                             progress.setProgress(++processedTransactionCount);
 287                             if (maxTransactionId < transactionId)
 288                                 maxTransactionId = transactionId;
 289                             if ((progress.isIndeterminate()) ||
 290                                 (progress.getTotal() < transactionId))
 291                                 progress.setTotal(transactionId);
 292                             publishProgress(progress);
 293                         }
 294                         m = reEnd.matcher(line);
 295                         if (m.find()) {
 296                             L("--- transaction value complete ---");
 297                             break LINES;
 298                         }
 299                         break;
 300
 301                     case EXPECTING_TRANSACTION_DESCRIPTION:
 302                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 303                             continue;
 304                         m = reTransactionDescription.matcher(line);
 305                         if (m.find()) {
 306                             if (transactionId == 0)
 307                                 throw new TransactionParserException(
 308                                         "Transaction Id is 0 while expecting description");
 309
 310                             String date = Objects.requireNonNull(m.group(1));
 311                             try {
 312                                 int equalsIndex = date.indexOf('=');
 313                                 if (equalsIndex >= 0)
 314                                     date = date.substring(equalsIndex + 1);
 315                                 transaction =
 316                                         new LedgerTransaction(transactionId, date, m.group(2));
 317                             }
 318                             catch (ParseException e) {
 319                                 throw new TransactionParserException(
 320                                         String.format("Error parsing date '%s'", date));
 321                             }
 322                             state = ParserState.EXPECTING_TRANSACTION_DETAILS;
 323                             L(String.format(Locale.ENGLISH,
 324                                     "transaction %d created for %s (%s) →" + " expecting details",
 325                                     transactionId, date, m.group(2)));
 326                         }
 327                         break;
 328
 329                     case EXPECTING_TRANSACTION_DETAILS:
 330                         if (line.isEmpty()) {
 331                             // transaction data collected
 332
 333                             transaction.finishLoading();
 334                             transactions.add(transaction);
 335
 336                             state = ParserState.EXPECTING_TRANSACTION;
 337                             L(String.format("transaction %s parsed → expecting transaction",
 338                                     transaction.getLedgerId()));
 339
 340 // sounds like a good idea, but transaction-1 may not be the first one chronologically
 341 // for example, when you add the initial seeding transaction after entering some others
 342 //                                            if (transactionId == 1) {
 343 //                                                L("This was the initial transaction.
 344 //                                                Terminating " +
 345 //                                                  "parser");
 346 //                                                break LINES;
 347 //                                            }
 348                         }
 349                         else {
 350                             LedgerTransactionAccount lta = parseTransactionAccountLine(line);
 351                             if (lta != null) {
 352                                 transaction.addAccount(lta);
 353                                 L(String.format(Locale.ENGLISH, "%d: %s = %s",
 354                                         transaction.getLedgerId(), lta.getAccountName(),
 355                                         lta.getAmount()));
 356                             }
 357                             else
 358                                 throw new IllegalStateException(
 359                                         String.format("Can't parse transaction %d details: %s",
 360                                                 transactionId, line));
 361                         }
 362                         break;
 363                     default:
 364                         throw new RuntimeException(
 365                                 String.format("Unknown parser updating %s", state.name()));
 366                 }
 367             }
 368
 369             throwIfCancelled();
 370         }
 371     }
 372     @NonNull
 373     public LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
 374                                              ArrayList<LedgerAccount> createdAccounts) {
 375         LedgerAccount acc = map.get(accountName);
 376
 377         if (acc != null)
 378             return acc;
 379
 380         String parentName = LedgerAccount.extractParentName(accountName);
 381         LedgerAccount parentAccount;
 382         if (parentName != null) {
 383             parentAccount = ensureAccountExists(parentName, map, createdAccounts);
 384         }
 385         else {
 386             parentAccount = null;
 387         }
 388
 389         acc = new LedgerAccount(accountName, parentAccount);
 390         createdAccounts.add(acc);
 391         return acc;
 392     }
 393     public void addNumberOfPostings(int number) {
 394         expectedPostingsCount += number;
 395     }
 396     private List<LedgerAccount> retrieveAccountList()
 397             throws IOException, HTTPException, ApiNotSupportedException {
 398         final API apiVersion = API.valueOf(profile.getApiVersion());
 399         if (apiVersion.equals(API.auto)) {
 400             return retrieveAccountListAnyVersion();
 401         }
 402         else if (apiVersion.equals(API.html)) {
 403             Logger.debug("json",
 404                     "Declining using JSON API for /accounts with configured legacy API version");
 405             return null;
 406         }
 407         else {
 408             return retrieveAccountListForVersion(apiVersion);
 409         }
 410     }
 411     private List<LedgerAccount> retrieveAccountListAnyVersion()
 412             throws ApiNotSupportedException, IOException, HTTPException {
 413         for (API ver : API.allVersions) {
 414             try {
 415                 return retrieveAccountListForVersion(ver);
 416             }
 417             catch (JsonParseException | RuntimeJsonMappingException e) {
 418                 Logger.debug("json",
 419                         String.format(Locale.US, "Error during account list retrieval using API %s",
 420                                 ver.getDescription()), e);
 421             }
 422
 423         }
 424
 425         throw new ApiNotSupportedException();
 426     }
 427     private List<LedgerAccount> retrieveAccountListForVersion(API version)
 428             throws IOException, HTTPException {
 429         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
 430         http.setAllowUserInteraction(false);
 431         switch (http.getResponseCode()) {
 432             case 200:
 433                 break;
 434             case 404:
 435                 return null;
 436             default:
 437                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 438         }
 439         publishProgress(Progress.indeterminate());
 440         SQLiteDatabase db = App.getDatabase();
 441         ArrayList<LedgerAccount> list = new ArrayList<>();
 442         HashMap<String, LedgerAccount> map = new HashMap<>();
 443         throwIfCancelled();
 444         try (InputStream resp = http.getInputStream()) {
 445             throwIfCancelled();
 446             if (http.getResponseCode() != 200)
 447                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 448
 449             AccountListParser parser = AccountListParser.forApiVersion(version, resp);
 450             expectedPostingsCount = 0;
 451
 452             while (true) {
 453                 throwIfCancelled();
 454                 LedgerAccount acc = parser.nextAccount(this, map);
 455                 if (acc == null)
 456                     break;
 457                 list.add(acc);
 458             }
 459             throwIfCancelled();
 460         }
 461
 462         return list;
 463     }
 464     private List<LedgerTransaction> retrieveTransactionList()
 465             throws ParseException, HTTPException, IOException, ApiNotSupportedException {
 466         final API apiVersion = API.valueOf(profile.getApiVersion());
 467         if (apiVersion.equals(API.auto)) {
 468             return retrieveTransactionListAnyVersion();
 469         }
 470         else if (apiVersion.equals(API.html)) {
 471             Logger.debug("json",
 472                     "Declining using JSON API for /accounts with configured legacy API version");
 473             return null;
 474         }
 475         else {
 476             return retrieveTransactionListForVersion(apiVersion);
 477         }
 478
 479     }
 480     private List<LedgerTransaction> retrieveTransactionListAnyVersion()
 481             throws ApiNotSupportedException {
 482         for (API ver : API.allVersions) {
 483             try {
 484                 return retrieveTransactionListForVersion(ver);
 485             }
 486             catch (Exception e) {
 487                 Logger.debug("json",
 488                         String.format(Locale.US, "Error during account list retrieval using API %s",
 489                                 ver.getDescription()));
 490             }
 491
 492         }
 493
 494         throw new ApiNotSupportedException();
 495     }
 496     private List<LedgerTransaction> retrieveTransactionListForVersion(API apiVersion)
 497             throws IOException, ParseException, HTTPException {
 498         Progress progress = new Progress();
 499         progress.setTotal(expectedPostingsCount);
 500
 501         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
 502         http.setAllowUserInteraction(false);
 503         publishProgress(progress);
 504         switch (http.getResponseCode()) {
 505             case 200:
 506                 break;
 507             case 404:
 508                 return null;
 509             default:
 510                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 511         }
 512         ArrayList<LedgerTransaction> trList = new ArrayList<>();
 513         try (InputStream resp = http.getInputStream()) {
 514             throwIfCancelled();
 515
 516             TransactionListParser parser = TransactionListParser.forApiVersion(apiVersion, resp);
 517
 518             int processedPostings = 0;
 519
 520             while (true) {
 521                 throwIfCancelled();
 522                 LedgerTransaction transaction = parser.nextTransaction();
 523                 throwIfCancelled();
 524                 if (transaction == null)
 525                     break;
 526
 527                 trList.add(transaction);
 528
 529                 progress.setProgress(processedPostings += transaction.getAccounts()
 530                                                                      .size());
 531 //                Logger.debug("trParser",
 532 //                        String.format(Locale.US, "Parsed transaction %d - %s", transaction
 533 //                        .getId(),
 534 //                                transaction.getDescription()));
 535 //                for (LedgerTransactionAccount acc : transaction.getAccounts()) {
 536 //                    Logger.debug("trParser",
 537 //                            String.format(Locale.US, "  %s", acc.getAccountName()));
 538 //                }
 539                 publishProgress(progress);
 540             }
 541
 542             throwIfCancelled();
 543         }
 544
 545         // json interface returns transactions if file order and the rest of the machinery
 546         // expects them in reverse chronological order
 547         Collections.sort(trList, (o1, o2) -> {
 548             int res = o2.getDate()
 549                         .compareTo(o1.getDate());
 550             if (res != 0)
 551                 return res;
 552             return Long.compare(o2.getLedgerId(), o1.getLedgerId());
 553         });
 554         return trList;
 555     }
 556
 557     @SuppressLint("DefaultLocale")
 558     @Override
 559     protected Result doInBackground(Void... params) {
 560         Data.backgroundTaskStarted();
 561         List<LedgerAccount> accounts;
 562         List<LedgerTransaction> transactions;
 563         try {
 564             accounts = retrieveAccountList();
 565             // accounts is null in API-version auto-detection and means
 566             // requesting 'html' API version via the JSON classes
 567             // this can't work, and the null results in the legacy code below
 568             // being called
 569             if (accounts == null)
 570                 transactions = null;
 571             else
 572                 transactions = retrieveTransactionList();
 573
 574             if (accounts == null || transactions == null) {
 575                 accounts = new ArrayList<>();
 576                 transactions = new ArrayList<>();
 577                 retrieveTransactionListLegacy(accounts, transactions);
 578             }
 579
 580             storeAccountsAndTransactions(accounts, transactions);
 581
 582             mainModel.updateDisplayedTransactionsFromWeb(transactions);
 583
 584             return new Result(accounts, transactions);
 585         }
 586         catch (MalformedURLException e) {
 587             e.printStackTrace();
 588             return new Result("Invalid server URL");
 589         }
 590         catch (HTTPException e) {
 591             e.printStackTrace();
 592             return new Result(
 593                     String.format("HTTP error %d: %s", e.getResponseCode(), e.getMessage()));
 594         }
 595         catch (IOException e) {
 596             e.printStackTrace();
 597             return new Result(e.getLocalizedMessage());
 598         }
 599         catch (RuntimeJsonMappingException e) {
 600             e.printStackTrace();
 601             return new Result(Result.ERR_JSON_PARSER_ERROR);
 602         }
 603         catch (ParseException e) {
 604             e.printStackTrace();
 605             return new Result("Network error");
 606         }
 607         catch (OperationCanceledException e) {
 608             e.printStackTrace();
 609             return new Result("Operation cancelled");
 610         }
 611         catch (ApiNotSupportedException e) {
 612             e.printStackTrace();
 613             return new Result("Server version not supported");
 614         }
 615         finally {
 616             Data.backgroundTaskFinished();
 617         }
 618     }
 619     @Transaction
 620     private void storeAccountsAndTransactions(List<LedgerAccount> accounts,
 621                                               List<LedgerTransaction> transactions) {
 622         AccountDAO accDao = DB.get()
 623                               .getAccountDAO();
 624         TransactionDAO trDao = DB.get()
 625                                  .getTransactionDAO();
 626         TransactionAccountDAO trAccDao = DB.get()
 627                                            .getTransactionAccountDAO();
 628         AccountValueDAO valDao = DB.get()
 629                                    .getAccountValueDAO();
 630
 631         final List<AccountWithAmounts> list = new ArrayList<>();
 632         for (LedgerAccount acc : accounts) {
 633             final AccountWithAmounts a = acc.toDBOWithAmounts();
 634             Account existing = accDao.getByNameSync(profile.getId(), acc.getName());
 635             if (existing != null) {
 636                 a.account.setExpanded(existing.isExpanded());
 637                 a.account.setAmountsExpanded(existing.isAmountsExpanded());
 638                 a.account.setId(
 639                         existing.getId()); // not strictly needed, but since we have it anyway...
 640             }
 641
 642             list.add(a);
 643         }
 644         accDao.storeAccountsSync(list, profile.getId());
 645
 646         long trGen = trDao.getGenerationSync(profile.getId());
 647         for (LedgerTransaction tr : transactions) {
 648             TransactionWithAccounts tran = tr.toDBO();
 649             tran.transaction.setGeneration(trGen);
 650             tran.transaction.setProfileId(profile.getId());
 651
 652             tran.transaction.setId(trDao.insertSync(tran.transaction));
 653
 654             for (TransactionAccount trAcc : tran.accounts) {
 655                 trAcc.setGeneration(trGen);
 656                 trAcc.setTransactionId(tran.transaction.getId());
 657                 trAcc.setId(trAccDao.insertSync(trAcc));
 658             }
 659         }
 660
 661         trDao.purgeOldTransactionsSync(profile.getId(), trGen);
 662
 663         DB.get()
 664           .getOptionDAO()
 665           .insertSync(new Option(profile.getId(), MLDB.OPT_LAST_SCRAPE,
 666                   String.valueOf((new Date()).getTime())));
 667     }
 668     public void throwIfCancelled() {
 669         if (isCancelled())
 670             throw new OperationCanceledException(null);
 671     }
 672     private enum ParserState {
 673         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
 674         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
 675     }
 676
 677     public enum ProgressState {STARTING, RUNNING, FINISHED}
 678
 679     public static class Progress {
 680         private int progress;
 681         private int total;
 682         private ProgressState state = ProgressState.RUNNING;
 683         private String error = null;
 684         private boolean indeterminate;
 685         Progress() {
 686             indeterminate = true;
 687         }
 688         Progress(int progress, int total) {
 689             this.indeterminate = false;
 690             this.progress = progress;
 691             this.total = total;
 692         }
 693         public static Progress indeterminate() {
 694             return new Progress();
 695         }
 696         public static Progress finished(String error) {
 697             Progress p = new Progress();
 698             p.setState(ProgressState.FINISHED);
 699             p.setError(error);
 700             return p;
 701         }
 702         public int getProgress() {
 703             ensureState(ProgressState.RUNNING);
 704             return progress;
 705         }
 706         protected void setProgress(int progress) {
 707             this.progress = progress;
 708             this.state = ProgressState.RUNNING;
 709         }
 710         public int getTotal() {
 711             ensureState(ProgressState.RUNNING);
 712             return total;
 713         }
 714         protected void setTotal(int total) {
 715             this.total = total;
 716             state = ProgressState.RUNNING;
 717             indeterminate = total == -1;
 718         }
 719         private void ensureState(ProgressState wanted) {
 720             if (state != wanted)
 721                 throw new IllegalStateException(
 722                         String.format("Bad state: %s, expected %s", state, wanted));
 723         }
 724         public ProgressState getState() {
 725             return state;
 726         }
 727         public void setState(ProgressState state) {
 728             this.state = state;
 729         }
 730         public String getError() {
 731             ensureState(ProgressState.FINISHED);
 732             return error;
 733         }
 734         public void setError(String error) {
 735             this.error = error;
 736             state = ProgressState.FINISHED;
 737         }
 738         public boolean isIndeterminate() {
 739             return indeterminate;
 740         }
 741         public void setIndeterminate(boolean indeterminate) {
 742             this.indeterminate = indeterminate;
 743         }
 744     }
 745
 746     private static class TransactionParserException extends IllegalStateException {
 747         TransactionParserException(String message) {
 748             super(message);
 749         }
 750     }
 751
 752     public static class Result {
 753         public static String ERR_JSON_PARSER_ERROR = "err_json_parser";
 754         public String error;
 755         public List<LedgerAccount> accounts;
 756         public List<LedgerTransaction> transactions;
 757         Result(String error) {
 758             this.error = error;
 759         }
 760         Result(List<LedgerAccount> accounts, List<LedgerTransaction> transactions) {
 761             this.accounts = accounts;
 762             this.transactions = transactions;
 763         }
 764     }
 765 }