app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java

   1 /*
   2  * Copyright © 2021 Damyan Ivanov.
   3  * This file is part of MoLe.
   4  * MoLe is free software: you can distribute it and/or modify it
   5  * under the term of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your opinion), any later version.
   8  *
   9  * MoLe is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12  * GNU General Public License terms for details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
  16  */
  17
  18 package net.ktnx.mobileledger.async;
  19
  20 import android.annotation.SuppressLint;
  21 import android.os.AsyncTask;
  22 import android.os.OperationCanceledException;
  23
  24 import androidx.annotation.NonNull;
  25 import androidx.room.Transaction;
  26
  27 import com.fasterxml.jackson.core.JsonParseException;
  28 import com.fasterxml.jackson.databind.RuntimeJsonMappingException;
  29
  30 import net.ktnx.mobileledger.dao.AccountDAO;
  31 import net.ktnx.mobileledger.dao.AccountValueDAO;
  32 import net.ktnx.mobileledger.dao.TransactionAccountDAO;
  33 import net.ktnx.mobileledger.dao.TransactionDAO;
  34 import net.ktnx.mobileledger.db.Account;
  35 import net.ktnx.mobileledger.db.AccountWithAmounts;
  36 import net.ktnx.mobileledger.db.DB;
  37 import net.ktnx.mobileledger.db.Option;
  38 import net.ktnx.mobileledger.db.Profile;
  39 import net.ktnx.mobileledger.db.TransactionAccount;
  40 import net.ktnx.mobileledger.db.TransactionWithAccounts;
  41 import net.ktnx.mobileledger.err.HTTPException;
  42 import net.ktnx.mobileledger.json.API;
  43 import net.ktnx.mobileledger.json.AccountListParser;
  44 import net.ktnx.mobileledger.json.ApiNotSupportedException;
  45 import net.ktnx.mobileledger.json.TransactionListParser;
  46 import net.ktnx.mobileledger.model.Data;
  47 import net.ktnx.mobileledger.model.LedgerAccount;
  48 import net.ktnx.mobileledger.model.LedgerTransaction;
  49 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
  50 import net.ktnx.mobileledger.ui.MainModel;
  51 import net.ktnx.mobileledger.utils.Logger;
  52 import net.ktnx.mobileledger.utils.NetworkUtil;
  53
  54 import java.io.BufferedReader;
  55 import java.io.IOException;
  56 import java.io.InputStream;
  57 import java.io.InputStreamReader;
  58 import java.net.HttpURLConnection;
  59 import java.net.MalformedURLException;
  60 import java.net.URLDecoder;
  61 import java.nio.charset.StandardCharsets;
  62 import java.text.ParseException;
  63 import java.util.ArrayList;
  64 import java.util.Collections;
  65 import java.util.Date;
  66 import java.util.HashMap;
  67 import java.util.List;
  68 import java.util.Locale;
  69 import java.util.Objects;
  70 import java.util.regex.Matcher;
  71 import java.util.regex.Pattern;
  72
  73
  74 public class RetrieveTransactionsTask extends
  75         AsyncTask<Void, RetrieveTransactionsTask.Progress, RetrieveTransactionsTask.Result> {
  76     private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
  77     private static final Pattern reComment = Pattern.compile("^\\s*;");
  78     private static final Pattern reTransactionStart = Pattern.compile(
  79             "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
  80             "\"[^\"]*>([\\d.-]+)</td>");
  81     private static final Pattern reTransactionDescription =
  82             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
  83     private static final Pattern reTransactionDetails = Pattern.compile(
  84             "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
  85             "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
  86     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
  87     private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
  88     private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
  89     // %3A is '='
  90     private final Pattern reAccountName =
  91             Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
  92     private final Pattern reAccountValue = Pattern.compile(
  93             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
  94     private final MainModel mainModel;
  95     private final Profile profile;
  96     private int expectedPostingsCount = -1;
  97     public RetrieveTransactionsTask(@NonNull MainModel mainModel, @NonNull Profile profile) {
  98         this.mainModel = mainModel;
  99         this.profile = profile;
 100     }
 101     private static void L(String msg) {
 102         //debug("transaction-parser", msg);
 103     }
 104     static LedgerTransactionAccount parseTransactionAccountLine(String line) {
 105         Matcher m = reTransactionDetails.matcher(line);
 106         if (m.find()) {
 107             String postingStatus = m.group(1);
 108             String acc_name = m.group(2);
 109             String currencyPre = m.group(3);
 110             String amount = Objects.requireNonNull(m.group(4));
 111             String currencyPost = m.group(5);
 112
 113             String currency = null;
 114             if ((currencyPre != null) && (currencyPre.length() > 0)) {
 115                 if ((currencyPost != null) && (currencyPost.length() > 0))
 116                     return null;
 117                 currency = currencyPre;
 118             }
 119             else if ((currencyPost != null) && (currencyPost.length() > 0)) {
 120                 currency = currencyPost;
 121             }
 122
 123             amount = amount.replace(',', '.');
 124
 125             return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
 126         }
 127         else {
 128             return null;
 129         }
 130     }
 131     @Override
 132     protected void onProgressUpdate(Progress... values) {
 133         super.onProgressUpdate(values);
 134         Data.backgroundTaskProgress.postValue(values[0]);
 135     }
 136     @Override
 137     protected void onPostExecute(Result result) {
 138         super.onPostExecute(result);
 139         Progress progress = new Progress();
 140         progress.setState(ProgressState.FINISHED);
 141         progress.setError(result.error);
 142         onProgressUpdate(progress);
 143     }
 144     @Override
 145     protected void onCancelled() {
 146         super.onCancelled();
 147         Progress progress = new Progress();
 148         progress.setState(ProgressState.FINISHED);
 149         onProgressUpdate(progress);
 150     }
 151     private void retrieveTransactionListLegacy(List<LedgerAccount> accounts,
 152                                                List<LedgerTransaction> transactions)
 153             throws IOException, HTTPException {
 154         Progress progress = Progress.indeterminate();
 155         progress.setState(ProgressState.RUNNING);
 156         progress.setTotal(expectedPostingsCount);
 157         int maxTransactionId = -1;
 158         HashMap<String, LedgerAccount> map = new HashMap<>();
 159         LedgerAccount lastAccount = null;
 160         ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
 161
 162         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
 163         http.setAllowUserInteraction(false);
 164         publishProgress(progress);
 165         if (http.getResponseCode() != 200)
 166             throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 167
 168         try (InputStream resp = http.getInputStream()) {
 169             if (http.getResponseCode() != 200)
 170                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 171
 172             int matchedTransactionsCount = 0;
 173
 174             ParserState state = ParserState.EXPECTING_ACCOUNT;
 175             String line;
 176             BufferedReader buf =
 177                     new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
 178
 179             int processedTransactionCount = 0;
 180             int transactionId = 0;
 181             LedgerTransaction transaction = null;
 182             LINES:
 183             while ((line = buf.readLine()) != null) {
 184                 throwIfCancelled();
 185                 Matcher m;
 186                 m = reComment.matcher(line);
 187                 if (m.find()) {
 188                     // TODO: comments are ignored for now
 189 //                            Log.v("transaction-parser", "Ignoring comment");
 190                     continue;
 191                 }
 192                 //L(String.format("State is %d", updating));
 193                 switch (state) {
 194                     case EXPECTING_ACCOUNT:
 195                         if (line.equals("<h2>General Journal</h2>")) {
 196                             state = ParserState.EXPECTING_TRANSACTION;
 197                             L("→ expecting transaction");
 198                             continue;
 199                         }
 200                         m = reAccountName.matcher(line);
 201                         if (m.find()) {
 202                             String acct_encoded = m.group(1);
 203                             String accName = URLDecoder.decode(acct_encoded, "UTF-8");
 204                             accName = accName.replace("\"", "");
 205                             L(String.format("found account: %s", accName));
 206
 207                             lastAccount = map.get(accName);
 208                             if (lastAccount != null) {
 209                                 L(String.format("ignoring duplicate account '%s'", accName));
 210                                 continue;
 211                             }
 212                             String parentAccountName = LedgerAccount.extractParentName(accName);
 213                             LedgerAccount parentAccount;
 214                             if (parentAccountName != null) {
 215                                 parentAccount = ensureAccountExists(parentAccountName, map,
 216                                         syntheticAccounts);
 217                             }
 218                             else {
 219                                 parentAccount = null;
 220                             }
 221                             lastAccount = new LedgerAccount(accName, parentAccount);
 222
 223                             accounts.add(lastAccount);
 224                             map.put(accName, lastAccount);
 225
 226                             state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
 227                             L("→ expecting account amount");
 228                         }
 229                         break;
 230
 231                     case EXPECTING_ACCOUNT_AMOUNT:
 232                         m = reAccountValue.matcher(line);
 233                         boolean match_found = false;
 234                         while (m.find()) {
 235                             throwIfCancelled();
 236
 237                             match_found = true;
 238                             String value = Objects.requireNonNull(m.group(1));
 239                             String currency = m.group(2);
 240                             if (currency == null)
 241                                 currency = "";
 242
 243                             {
 244                                 Matcher tmpM = reDecimalComma.matcher(value);
 245                                 if (tmpM.find()) {
 246                                     value = value.replace(".", "");
 247                                     value = value.replace(',', '.');
 248                                 }
 249
 250                                 tmpM = reDecimalPoint.matcher(value);
 251                                 if (tmpM.find()) {
 252                                     value = value.replace(",", "");
 253                                     value = value.replace(" ", "");
 254                                 }
 255                             }
 256                             L("curr=" + currency + ", value=" + value);
 257                             final float val = Float.parseFloat(value);
 258                             lastAccount.addAmount(val, currency);
 259                             for (LedgerAccount syn : syntheticAccounts) {
 260                                 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
 261                                         currency, val, syn.getName()));
 262                                 syn.addAmount(val, currency);
 263                             }
 264                         }
 265
 266                         if (match_found) {
 267                             syntheticAccounts.clear();
 268                             state = ParserState.EXPECTING_ACCOUNT;
 269                             L("→ expecting account");
 270                         }
 271
 272                         break;
 273
 274                     case EXPECTING_TRANSACTION:
 275                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 276                             continue;
 277                         m = reTransactionStart.matcher(line);
 278                         if (m.find()) {
 279                             transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
 280                             state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
 281                             L(String.format(Locale.ENGLISH,
 282                                     "found transaction %d → expecting description", transactionId));
 283                             progress.setProgress(++processedTransactionCount);
 284                             if (maxTransactionId < transactionId)
 285                                 maxTransactionId = transactionId;
 286                             if ((progress.isIndeterminate()) ||
 287                                 (progress.getTotal() < transactionId))
 288                                 progress.setTotal(transactionId);
 289                             publishProgress(progress);
 290                         }
 291                         m = reEnd.matcher(line);
 292                         if (m.find()) {
 293                             L("--- transaction value complete ---");
 294                             break LINES;
 295                         }
 296                         break;
 297
 298                     case EXPECTING_TRANSACTION_DESCRIPTION:
 299                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 300                             continue;
 301                         m = reTransactionDescription.matcher(line);
 302                         if (m.find()) {
 303                             if (transactionId == 0)
 304                                 throw new TransactionParserException(
 305                                         "Transaction Id is 0 while expecting description");
 306
 307                             String date = Objects.requireNonNull(m.group(1));
 308                             try {
 309                                 int equalsIndex = date.indexOf('=');
 310                                 if (equalsIndex >= 0)
 311                                     date = date.substring(equalsIndex + 1);
 312                                 transaction =
 313                                         new LedgerTransaction(transactionId, date, m.group(2));
 314                             }
 315                             catch (ParseException e) {
 316                                 throw new TransactionParserException(
 317                                         String.format("Error parsing date '%s'", date));
 318                             }
 319                             state = ParserState.EXPECTING_TRANSACTION_DETAILS;
 320                             L(String.format(Locale.ENGLISH,
 321                                     "transaction %d created for %s (%s) →" + " expecting details",
 322                                     transactionId, date, m.group(2)));
 323                         }
 324                         break;
 325
 326                     case EXPECTING_TRANSACTION_DETAILS:
 327                         if (line.isEmpty()) {
 328                             // transaction data collected
 329
 330                             transaction.finishLoading();
 331                             transactions.add(transaction);
 332
 333                             state = ParserState.EXPECTING_TRANSACTION;
 334                             L(String.format("transaction %s parsed → expecting transaction",
 335                                     transaction.getLedgerId()));
 336
 337 // sounds like a good idea, but transaction-1 may not be the first one chronologically
 338 // for example, when you add the initial seeding transaction after entering some others
 339 //                                            if (transactionId == 1) {
 340 //                                                L("This was the initial transaction.
 341 //                                                Terminating " +
 342 //                                                  "parser");
 343 //                                                break LINES;
 344 //                                            }
 345                         }
 346                         else {
 347                             LedgerTransactionAccount lta = parseTransactionAccountLine(line);
 348                             if (lta != null) {
 349                                 transaction.addAccount(lta);
 350                                 L(String.format(Locale.ENGLISH, "%d: %s = %s",
 351                                         transaction.getLedgerId(), lta.getAccountName(),
 352                                         lta.getAmount()));
 353                             }
 354                             else
 355                                 throw new IllegalStateException(
 356                                         String.format("Can't parse transaction %d details: %s",
 357                                                 transactionId, line));
 358                         }
 359                         break;
 360                     default:
 361                         throw new RuntimeException(
 362                                 String.format("Unknown parser updating %s", state.name()));
 363                 }
 364             }
 365
 366             throwIfCancelled();
 367         }
 368     }
 369     @NonNull
 370     public LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
 371                                              ArrayList<LedgerAccount> createdAccounts) {
 372         LedgerAccount acc = map.get(accountName);
 373
 374         if (acc != null)
 375             return acc;
 376
 377         String parentName = LedgerAccount.extractParentName(accountName);
 378         LedgerAccount parentAccount;
 379         if (parentName != null) {
 380             parentAccount = ensureAccountExists(parentName, map, createdAccounts);
 381         }
 382         else {
 383             parentAccount = null;
 384         }
 385
 386         acc = new LedgerAccount(accountName, parentAccount);
 387         createdAccounts.add(acc);
 388         return acc;
 389     }
 390     public void addNumberOfPostings(int number) {
 391         expectedPostingsCount += number;
 392     }
 393     private List<LedgerAccount> retrieveAccountList()
 394             throws IOException, HTTPException, ApiNotSupportedException {
 395         final API apiVersion = API.valueOf(profile.getApiVersion());
 396         if (apiVersion.equals(API.auto)) {
 397             return retrieveAccountListAnyVersion();
 398         }
 399         else if (apiVersion.equals(API.html)) {
 400             Logger.debug("json",
 401                     "Declining using JSON API for /accounts with configured legacy API version");
 402             return null;
 403         }
 404         else {
 405             return retrieveAccountListForVersion(apiVersion);
 406         }
 407     }
 408     private List<LedgerAccount> retrieveAccountListAnyVersion()
 409             throws ApiNotSupportedException, IOException, HTTPException {
 410         for (API ver : API.allVersions) {
 411             try {
 412                 return retrieveAccountListForVersion(ver);
 413             }
 414             catch (JsonParseException | RuntimeJsonMappingException e) {
 415                 Logger.debug("json",
 416                         String.format(Locale.US, "Error during account list retrieval using API %s",
 417                                 ver.getDescription()), e);
 418             }
 419
 420         }
 421
 422         throw new ApiNotSupportedException();
 423     }
 424     private List<LedgerAccount> retrieveAccountListForVersion(API version)
 425             throws IOException, HTTPException {
 426         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
 427         http.setAllowUserInteraction(false);
 428         switch (http.getResponseCode()) {
 429             case 200:
 430                 break;
 431             case 404:
 432                 return null;
 433             default:
 434                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 435         }
 436         publishProgress(Progress.indeterminate());
 437         ArrayList<LedgerAccount> list = new ArrayList<>();
 438         HashMap<String, LedgerAccount> map = new HashMap<>();
 439         throwIfCancelled();
 440         try (InputStream resp = http.getInputStream()) {
 441             throwIfCancelled();
 442             if (http.getResponseCode() != 200)
 443                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 444
 445             AccountListParser parser = AccountListParser.forApiVersion(version, resp);
 446             expectedPostingsCount = 0;
 447
 448             while (true) {
 449                 throwIfCancelled();
 450                 LedgerAccount acc = parser.nextAccount(this, map);
 451                 if (acc == null)
 452                     break;
 453                 list.add(acc);
 454             }
 455             throwIfCancelled();
 456         }
 457
 458         return list;
 459     }
 460     private List<LedgerTransaction> retrieveTransactionList()
 461             throws ParseException, HTTPException, IOException, ApiNotSupportedException {
 462         final API apiVersion = API.valueOf(profile.getApiVersion());
 463         if (apiVersion.equals(API.auto)) {
 464             return retrieveTransactionListAnyVersion();
 465         }
 466         else if (apiVersion.equals(API.html)) {
 467             Logger.debug("json",
 468                     "Declining using JSON API for /accounts with configured legacy API version");
 469             return null;
 470         }
 471         else {
 472             return retrieveTransactionListForVersion(apiVersion);
 473         }
 474
 475     }
 476     private List<LedgerTransaction> retrieveTransactionListAnyVersion()
 477             throws ApiNotSupportedException {
 478         for (API ver : API.allVersions) {
 479             try {
 480                 return retrieveTransactionListForVersion(ver);
 481             }
 482             catch (Exception e) {
 483                 Logger.debug("json",
 484                         String.format(Locale.US, "Error during account list retrieval using API %s",
 485                                 ver.getDescription()));
 486             }
 487
 488         }
 489
 490         throw new ApiNotSupportedException();
 491     }
 492     private List<LedgerTransaction> retrieveTransactionListForVersion(API apiVersion)
 493             throws IOException, ParseException, HTTPException {
 494         Progress progress = new Progress();
 495         progress.setTotal(expectedPostingsCount);
 496
 497         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
 498         http.setAllowUserInteraction(false);
 499         publishProgress(progress);
 500         switch (http.getResponseCode()) {
 501             case 200:
 502                 break;
 503             case 404:
 504                 return null;
 505             default:
 506                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 507         }
 508         ArrayList<LedgerTransaction> trList = new ArrayList<>();
 509         try (InputStream resp = http.getInputStream()) {
 510             throwIfCancelled();
 511
 512             TransactionListParser parser = TransactionListParser.forApiVersion(apiVersion, resp);
 513
 514             int processedPostings = 0;
 515
 516             while (true) {
 517                 throwIfCancelled();
 518                 LedgerTransaction transaction = parser.nextTransaction();
 519                 throwIfCancelled();
 520                 if (transaction == null)
 521                     break;
 522
 523                 trList.add(transaction);
 524
 525                 progress.setProgress(processedPostings += transaction.getAccounts()
 526                                                                      .size());
 527 //                Logger.debug("trParser",
 528 //                        String.format(Locale.US, "Parsed transaction %d - %s", transaction
 529 //                        .getId(),
 530 //                                transaction.getDescription()));
 531 //                for (LedgerTransactionAccount acc : transaction.getAccounts()) {
 532 //                    Logger.debug("trParser",
 533 //                            String.format(Locale.US, "  %s", acc.getAccountName()));
 534 //                }
 535                 publishProgress(progress);
 536             }
 537
 538             throwIfCancelled();
 539         }
 540
 541         // json interface returns transactions if file order and the rest of the machinery
 542         // expects them in reverse chronological order
 543         Collections.sort(trList, (o1, o2) -> {
 544             int res = o2.getDate()
 545                         .compareTo(o1.getDate());
 546             if (res != 0)
 547                 return res;
 548             return Long.compare(o2.getLedgerId(), o1.getLedgerId());
 549         });
 550         return trList;
 551     }
 552
 553     @SuppressLint("DefaultLocale")
 554     @Override
 555     protected Result doInBackground(Void... params) {
 556         Data.backgroundTaskStarted();
 557         List<LedgerAccount> accounts;
 558         List<LedgerTransaction> transactions;
 559         try {
 560             accounts = retrieveAccountList();
 561             // accounts is null in API-version auto-detection and means
 562             // requesting 'html' API version via the JSON classes
 563             // this can't work, and the null results in the legacy code below
 564             // being called
 565             if (accounts == null)
 566                 transactions = null;
 567             else
 568                 transactions = retrieveTransactionList();
 569
 570             if (accounts == null || transactions == null) {
 571                 accounts = new ArrayList<>();
 572                 transactions = new ArrayList<>();
 573                 retrieveTransactionListLegacy(accounts, transactions);
 574             }
 575
 576             storeAccountsAndTransactions(accounts, transactions);
 577
 578             mainModel.updateDisplayedTransactionsFromWeb(transactions);
 579
 580             return new Result(accounts, transactions);
 581         }
 582         catch (MalformedURLException e) {
 583             e.printStackTrace();
 584             return new Result("Invalid server URL");
 585         }
 586         catch (HTTPException e) {
 587             e.printStackTrace();
 588             return new Result(
 589                     String.format("HTTP error %d: %s", e.getResponseCode(), e.getMessage()));
 590         }
 591         catch (IOException e) {
 592             e.printStackTrace();
 593             return new Result(e.getLocalizedMessage());
 594         }
 595         catch (RuntimeJsonMappingException e) {
 596             e.printStackTrace();
 597             return new Result(Result.ERR_JSON_PARSER_ERROR);
 598         }
 599         catch (ParseException e) {
 600             e.printStackTrace();
 601             return new Result("Network error");
 602         }
 603         catch (OperationCanceledException e) {
 604             e.printStackTrace();
 605             return new Result("Operation cancelled");
 606         }
 607         catch (ApiNotSupportedException e) {
 608             e.printStackTrace();
 609             return new Result("Server version not supported");
 610         }
 611         finally {
 612             Data.backgroundTaskFinished();
 613         }
 614     }
 615     @Transaction
 616     private void storeAccountsAndTransactions(List<LedgerAccount> accounts,
 617                                               List<LedgerTransaction> transactions) {
 618         AccountDAO accDao = DB.get()
 619                               .getAccountDAO();
 620         TransactionDAO trDao = DB.get()
 621                                  .getTransactionDAO();
 622         TransactionAccountDAO trAccDao = DB.get()
 623                                            .getTransactionAccountDAO();
 624         AccountValueDAO valDao = DB.get()
 625                                    .getAccountValueDAO();
 626
 627         final List<AccountWithAmounts> list = new ArrayList<>();
 628         for (LedgerAccount acc : accounts) {
 629             final AccountWithAmounts a = acc.toDBOWithAmounts();
 630             Account existing = accDao.getByNameSync(profile.getId(), acc.getName());
 631             if (existing != null) {
 632                 a.account.setExpanded(existing.isExpanded());
 633                 a.account.setAmountsExpanded(existing.isAmountsExpanded());
 634                 a.account.setId(
 635                         existing.getId()); // not strictly needed, but since we have it anyway...
 636             }
 637
 638             list.add(a);
 639         }
 640         accDao.storeAccountsSync(list, profile.getId());
 641
 642         long trGen = trDao.getGenerationSync(profile.getId());
 643         for (LedgerTransaction tr : transactions) {
 644             TransactionWithAccounts tran = tr.toDBO();
 645             tran.transaction.setGeneration(trGen);
 646             tran.transaction.setProfileId(profile.getId());
 647
 648             tran.transaction.setId(trDao.insertSync(tran.transaction));
 649
 650             for (TransactionAccount trAcc : tran.accounts) {
 651                 trAcc.setGeneration(trGen);
 652                 trAcc.setTransactionId(tran.transaction.getId());
 653                 trAcc.setId(trAccDao.insertSync(trAcc));
 654             }
 655         }
 656
 657         trDao.purgeOldTransactionsSync(profile.getId(), trGen);
 658
 659         DB.get()
 660           .getOptionDAO()
 661           .insertSync(new Option(profile.getId(), Option.OPT_LAST_SCRAPE,
 662                   String.valueOf((new Date()).getTime())));
 663     }
 664     public void throwIfCancelled() {
 665         if (isCancelled())
 666             throw new OperationCanceledException(null);
 667     }
 668     private enum ParserState {
 669         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
 670         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
 671     }
 672
 673     public enum ProgressState {STARTING, RUNNING, FINISHED}
 674
 675     public static class Progress {
 676         private int progress;
 677         private int total;
 678         private ProgressState state = ProgressState.RUNNING;
 679         private String error = null;
 680         private boolean indeterminate;
 681         Progress() {
 682             indeterminate = true;
 683         }
 684         Progress(int progress, int total) {
 685             this.indeterminate = false;
 686             this.progress = progress;
 687             this.total = total;
 688         }
 689         public static Progress indeterminate() {
 690             return new Progress();
 691         }
 692         public static Progress finished(String error) {
 693             Progress p = new Progress();
 694             p.setState(ProgressState.FINISHED);
 695             p.setError(error);
 696             return p;
 697         }
 698         public int getProgress() {
 699             ensureState(ProgressState.RUNNING);
 700             return progress;
 701         }
 702         protected void setProgress(int progress) {
 703             this.progress = progress;
 704             this.state = ProgressState.RUNNING;
 705         }
 706         public int getTotal() {
 707             ensureState(ProgressState.RUNNING);
 708             return total;
 709         }
 710         protected void setTotal(int total) {
 711             this.total = total;
 712             state = ProgressState.RUNNING;
 713             indeterminate = total == -1;
 714         }
 715         private void ensureState(ProgressState wanted) {
 716             if (state != wanted)
 717                 throw new IllegalStateException(
 718                         String.format("Bad state: %s, expected %s", state, wanted));
 719         }
 720         public ProgressState getState() {
 721             return state;
 722         }
 723         public void setState(ProgressState state) {
 724             this.state = state;
 725         }
 726         public String getError() {
 727             ensureState(ProgressState.FINISHED);
 728             return error;
 729         }
 730         public void setError(String error) {
 731             this.error = error;
 732             state = ProgressState.FINISHED;
 733         }
 734         public boolean isIndeterminate() {
 735             return indeterminate;
 736         }
 737         public void setIndeterminate(boolean indeterminate) {
 738             this.indeterminate = indeterminate;
 739         }
 740     }
 741
 742     private static class TransactionParserException extends IllegalStateException {
 743         TransactionParserException(String message) {
 744             super(message);
 745         }
 746     }
 747
 748     public static class Result {
 749         public static String ERR_JSON_PARSER_ERROR = "err_json_parser";
 750         public String error;
 751         public List<LedgerAccount> accounts;
 752         public List<LedgerTransaction> transactions;
 753         Result(String error) {
 754             this.error = error;
 755         }
 756         Result(List<LedgerAccount> accounts, List<LedgerTransaction> transactions) {
 757             this.accounts = accounts;
 758             this.transactions = transactions;
 759         }
 760     }
 761 }