app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java

   1 /*
   2  * Copyright © 2021 Damyan Ivanov.
   3  * This file is part of MoLe.
   4  * MoLe is free software: you can distribute it and/or modify it
   5  * under the term of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your opinion), any later version.
   8  *
   9  * MoLe is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12  * GNU General Public License terms for details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
  16  */
  17
  18 package net.ktnx.mobileledger.async;
  19
  20 import android.annotation.SuppressLint;
  21 import android.os.OperationCanceledException;
  22
  23 import androidx.annotation.NonNull;
  24
  25 import com.fasterxml.jackson.core.JsonParseException;
  26 import com.fasterxml.jackson.databind.RuntimeJsonMappingException;
  27
  28 import net.ktnx.mobileledger.dao.AccountDAO;
  29 import net.ktnx.mobileledger.dao.TransactionDAO;
  30 import net.ktnx.mobileledger.db.Account;
  31 import net.ktnx.mobileledger.db.AccountWithAmounts;
  32 import net.ktnx.mobileledger.db.DB;
  33 import net.ktnx.mobileledger.db.Option;
  34 import net.ktnx.mobileledger.db.Profile;
  35 import net.ktnx.mobileledger.db.TransactionWithAccounts;
  36 import net.ktnx.mobileledger.err.HTTPException;
  37 import net.ktnx.mobileledger.json.API;
  38 import net.ktnx.mobileledger.json.AccountListParser;
  39 import net.ktnx.mobileledger.json.ApiNotSupportedException;
  40 import net.ktnx.mobileledger.json.TransactionListParser;
  41 import net.ktnx.mobileledger.model.Data;
  42 import net.ktnx.mobileledger.model.LedgerAccount;
  43 import net.ktnx.mobileledger.model.LedgerTransaction;
  44 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
  45 import net.ktnx.mobileledger.utils.Logger;
  46 import net.ktnx.mobileledger.utils.NetworkUtil;
  47
  48 import java.io.BufferedReader;
  49 import java.io.IOException;
  50 import java.io.InputStream;
  51 import java.io.InputStreamReader;
  52 import java.net.HttpURLConnection;
  53 import java.net.MalformedURLException;
  54 import java.net.URLDecoder;
  55 import java.nio.charset.StandardCharsets;
  56 import java.text.ParseException;
  57 import java.util.ArrayList;
  58 import java.util.Collections;
  59 import java.util.Date;
  60 import java.util.HashMap;
  61 import java.util.List;
  62 import java.util.Locale;
  63 import java.util.Objects;
  64 import java.util.regex.Matcher;
  65 import java.util.regex.Pattern;
  66
  67
  68 public class RetrieveTransactionsTask extends Thread {
  69     private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
  70     private static final Pattern reComment = Pattern.compile("^\\s*;");
  71     private static final Pattern reTransactionStart = Pattern.compile(
  72             "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
  73             "\"[^\"]*>([\\d.-]+)</td>");
  74     private static final Pattern reTransactionDescription =
  75             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
  76     private static final Pattern reTransactionDetails = Pattern.compile(
  77             "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
  78             "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
  79     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
  80     private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
  81     private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
  82     private static final String TAG = "RTT";
  83     // %3A is '='
  84     private final Pattern reAccountName =
  85             Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
  86     private final Pattern reAccountValue = Pattern.compile(
  87             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
  88     private final Profile profile;
  89     private int expectedPostingsCount = -1;
  90     public RetrieveTransactionsTask(@NonNull Profile profile) {
  91         this.profile = profile;
  92     }
  93     private static void L(String msg) {
  94         //debug("transaction-parser", msg);
  95     }
  96     static LedgerTransactionAccount parseTransactionAccountLine(String line) {
  97         Matcher m = reTransactionDetails.matcher(line);
  98         if (m.find()) {
  99             String postingStatus = m.group(1);
 100             String acc_name = m.group(2);
 101             String currencyPre = m.group(3);
 102             String amount = Objects.requireNonNull(m.group(4));
 103             String currencyPost = m.group(5);
 104
 105             String currency = null;
 106             if ((currencyPre != null) && (currencyPre.length() > 0)) {
 107                 if ((currencyPost != null) && (currencyPost.length() > 0))
 108                     return null;
 109                 currency = currencyPre;
 110             }
 111             else if ((currencyPost != null) && (currencyPost.length() > 0)) {
 112                 currency = currencyPost;
 113             }
 114
 115             amount = amount.replace(',', '.');
 116
 117             return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
 118         }
 119         else {
 120             return null;
 121         }
 122     }
 123     private void publishProgress(Progress progress) {
 124         Data.backgroundTaskProgress.postValue(progress);
 125     }
 126     private void finish(Result result) {
 127         Progress progress = new Progress();
 128         progress.setState(ProgressState.FINISHED);
 129         progress.setError(result.error);
 130         publishProgress(progress);
 131     }
 132     private void cancel() {
 133         Progress progress = new Progress();
 134         progress.setState(ProgressState.FINISHED);
 135         publishProgress(progress);
 136     }
 137     private void retrieveTransactionListLegacy(List<LedgerAccount> accounts,
 138                                                List<LedgerTransaction> transactions)
 139             throws IOException, HTTPException {
 140         Progress progress = Progress.indeterminate();
 141         progress.setState(ProgressState.RUNNING);
 142         progress.setTotal(expectedPostingsCount);
 143         int maxTransactionId = -1;
 144         HashMap<String, LedgerAccount> map = new HashMap<>();
 145         LedgerAccount lastAccount = null;
 146         ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
 147
 148         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
 149         http.setAllowUserInteraction(false);
 150         publishProgress(progress);
 151         if (http.getResponseCode() != 200)
 152             throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 153
 154         try (InputStream resp = http.getInputStream()) {
 155             if (http.getResponseCode() != 200)
 156                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 157
 158             int matchedTransactionsCount = 0;
 159
 160             ParserState state = ParserState.EXPECTING_ACCOUNT;
 161             String line;
 162             BufferedReader buf =
 163                     new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
 164
 165             int processedTransactionCount = 0;
 166             int transactionId = 0;
 167             LedgerTransaction transaction = null;
 168             LINES:
 169             while ((line = buf.readLine()) != null) {
 170                 throwIfCancelled();
 171                 Matcher m;
 172                 m = reComment.matcher(line);
 173                 if (m.find()) {
 174                     // TODO: comments are ignored for now
 175 //                            Log.v("transaction-parser", "Ignoring comment");
 176                     continue;
 177                 }
 178                 //L(String.format("State is %d", updating));
 179                 switch (state) {
 180                     case EXPECTING_ACCOUNT:
 181                         if (line.equals("<h2>General Journal</h2>")) {
 182                             state = ParserState.EXPECTING_TRANSACTION;
 183                             L("→ expecting transaction");
 184                             continue;
 185                         }
 186                         m = reAccountName.matcher(line);
 187                         if (m.find()) {
 188                             String acct_encoded = m.group(1);
 189                             String accName = URLDecoder.decode(acct_encoded, "UTF-8");
 190                             accName = accName.replace("\"", "");
 191                             L(String.format("found account: %s", accName));
 192
 193                             lastAccount = map.get(accName);
 194                             if (lastAccount != null) {
 195                                 L(String.format("ignoring duplicate account '%s'", accName));
 196                                 continue;
 197                             }
 198                             String parentAccountName = LedgerAccount.extractParentName(accName);
 199                             LedgerAccount parentAccount;
 200                             if (parentAccountName != null) {
 201                                 parentAccount = ensureAccountExists(parentAccountName, map,
 202                                         syntheticAccounts);
 203                             }
 204                             else {
 205                                 parentAccount = null;
 206                             }
 207                             lastAccount = new LedgerAccount(accName, parentAccount);
 208
 209                             accounts.add(lastAccount);
 210                             map.put(accName, lastAccount);
 211
 212                             state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
 213                             L("→ expecting account amount");
 214                         }
 215                         break;
 216
 217                     case EXPECTING_ACCOUNT_AMOUNT:
 218                         m = reAccountValue.matcher(line);
 219                         boolean match_found = false;
 220                         while (m.find()) {
 221                             throwIfCancelled();
 222
 223                             match_found = true;
 224                             String value = Objects.requireNonNull(m.group(1));
 225                             String currency = m.group(2);
 226                             if (currency == null)
 227                                 currency = "";
 228
 229                             {
 230                                 Matcher tmpM = reDecimalComma.matcher(value);
 231                                 if (tmpM.find()) {
 232                                     value = value.replace(".", "");
 233                                     value = value.replace(',', '.');
 234                                 }
 235
 236                                 tmpM = reDecimalPoint.matcher(value);
 237                                 if (tmpM.find()) {
 238                                     value = value.replace(",", "");
 239                                     value = value.replace(" ", "");
 240                                 }
 241                             }
 242                             L("curr=" + currency + ", value=" + value);
 243                             final float val = Float.parseFloat(value);
 244                             lastAccount.addAmount(val, currency);
 245                             for (LedgerAccount syn : syntheticAccounts) {
 246                                 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
 247                                         currency, val, syn.getName()));
 248                                 syn.addAmount(val, currency);
 249                             }
 250                         }
 251
 252                         if (match_found) {
 253                             syntheticAccounts.clear();
 254                             state = ParserState.EXPECTING_ACCOUNT;
 255                             L("→ expecting account");
 256                         }
 257
 258                         break;
 259
 260                     case EXPECTING_TRANSACTION:
 261                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 262                             continue;
 263                         m = reTransactionStart.matcher(line);
 264                         if (m.find()) {
 265                             transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
 266                             state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
 267                             L(String.format(Locale.ENGLISH,
 268                                     "found transaction %d → expecting description", transactionId));
 269                             progress.setProgress(++processedTransactionCount);
 270                             if (maxTransactionId < transactionId)
 271                                 maxTransactionId = transactionId;
 272                             if ((progress.isIndeterminate()) ||
 273                                 (progress.getTotal() < transactionId))
 274                                 progress.setTotal(transactionId);
 275                             publishProgress(progress);
 276                         }
 277                         m = reEnd.matcher(line);
 278                         if (m.find()) {
 279                             L("--- transaction value complete ---");
 280                             break LINES;
 281                         }
 282                         break;
 283
 284                     case EXPECTING_TRANSACTION_DESCRIPTION:
 285                         if (!line.isEmpty() && (line.charAt(0) == ' '))
 286                             continue;
 287                         m = reTransactionDescription.matcher(line);
 288                         if (m.find()) {
 289                             if (transactionId == 0)
 290                                 throw new TransactionParserException(
 291                                         "Transaction Id is 0 while expecting description");
 292
 293                             String date = Objects.requireNonNull(m.group(1));
 294                             try {
 295                                 int equalsIndex = date.indexOf('=');
 296                                 if (equalsIndex >= 0)
 297                                     date = date.substring(equalsIndex + 1);
 298                                 transaction =
 299                                         new LedgerTransaction(transactionId, date, m.group(2));
 300                             }
 301                             catch (ParseException e) {
 302                                 throw new TransactionParserException(
 303                                         String.format("Error parsing date '%s'", date));
 304                             }
 305                             state = ParserState.EXPECTING_TRANSACTION_DETAILS;
 306                             L(String.format(Locale.ENGLISH,
 307                                     "transaction %d created for %s (%s) →" + " expecting details",
 308                                     transactionId, date, m.group(2)));
 309                         }
 310                         break;
 311
 312                     case EXPECTING_TRANSACTION_DETAILS:
 313                         if (line.isEmpty()) {
 314                             // transaction data collected
 315
 316                             transaction.finishLoading();
 317                             transactions.add(transaction);
 318
 319                             state = ParserState.EXPECTING_TRANSACTION;
 320                             L(String.format("transaction %s parsed → expecting transaction",
 321                                     transaction.getLedgerId()));
 322
 323 // sounds like a good idea, but transaction-1 may not be the first one chronologically
 324 // for example, when you add the initial seeding transaction after entering some others
 325 //                                            if (transactionId == 1) {
 326 //                                                L("This was the initial transaction.
 327 //                                                Terminating " +
 328 //                                                  "parser");
 329 //                                                break LINES;
 330 //                                            }
 331                         }
 332                         else {
 333                             LedgerTransactionAccount lta = parseTransactionAccountLine(line);
 334                             if (lta != null) {
 335                                 transaction.addAccount(lta);
 336                                 L(String.format(Locale.ENGLISH, "%d: %s = %s",
 337                                         transaction.getLedgerId(), lta.getAccountName(),
 338                                         lta.getAmount()));
 339                             }
 340                             else
 341                                 throw new IllegalStateException(
 342                                         String.format("Can't parse transaction %d details: %s",
 343                                                 transactionId, line));
 344                         }
 345                         break;
 346                     default:
 347                         throw new RuntimeException(
 348                                 String.format("Unknown parser updating %s", state.name()));
 349                 }
 350             }
 351
 352             throwIfCancelled();
 353         }
 354     }
 355     @NonNull
 356     public LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
 357                                              ArrayList<LedgerAccount> createdAccounts) {
 358         LedgerAccount acc = map.get(accountName);
 359
 360         if (acc != null)
 361             return acc;
 362
 363         String parentName = LedgerAccount.extractParentName(accountName);
 364         LedgerAccount parentAccount;
 365         if (parentName != null) {
 366             parentAccount = ensureAccountExists(parentName, map, createdAccounts);
 367         }
 368         else {
 369             parentAccount = null;
 370         }
 371
 372         acc = new LedgerAccount(accountName, parentAccount);
 373         createdAccounts.add(acc);
 374         return acc;
 375     }
 376     public void addNumberOfPostings(int number) {
 377         expectedPostingsCount += number;
 378     }
 379     private List<LedgerAccount> retrieveAccountList()
 380             throws IOException, HTTPException, ApiNotSupportedException {
 381         final API apiVersion = API.valueOf(profile.getApiVersion());
 382         if (apiVersion.equals(API.auto)) {
 383             return retrieveAccountListAnyVersion();
 384         }
 385         else if (apiVersion.equals(API.html)) {
 386             Logger.debug("json",
 387                     "Declining using JSON API for /accounts with configured legacy API version");
 388             return null;
 389         }
 390         else {
 391             return retrieveAccountListForVersion(apiVersion);
 392         }
 393     }
 394     private List<LedgerAccount> retrieveAccountListAnyVersion()
 395             throws ApiNotSupportedException, IOException, HTTPException {
 396         for (API ver : API.allVersions) {
 397             try {
 398                 return retrieveAccountListForVersion(ver);
 399             }
 400             catch (JsonParseException | RuntimeJsonMappingException e) {
 401                 Logger.debug("json",
 402                         String.format(Locale.US, "Error during account list retrieval using API %s",
 403                                 ver.getDescription()), e);
 404             }
 405
 406         }
 407
 408         throw new ApiNotSupportedException();
 409     }
 410     private List<LedgerAccount> retrieveAccountListForVersion(API version)
 411             throws IOException, HTTPException {
 412         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
 413         http.setAllowUserInteraction(false);
 414         switch (http.getResponseCode()) {
 415             case 200:
 416                 break;
 417             case 404:
 418                 return null;
 419             default:
 420                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 421         }
 422         publishProgress(Progress.indeterminate());
 423         ArrayList<LedgerAccount> list = new ArrayList<>();
 424         HashMap<String, LedgerAccount> map = new HashMap<>();
 425         throwIfCancelled();
 426         try (InputStream resp = http.getInputStream()) {
 427             throwIfCancelled();
 428             if (http.getResponseCode() != 200)
 429                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
 430
 431             AccountListParser parser = AccountListParser.forApiVersion(version, resp);
 432             expectedPostingsCount = 0;
 433
 434             while (true) {
 435                 throwIfCancelled();
 436                 LedgerAccount acc = parser.nextAccount(this, map);
 437                 if (acc == null)
 438                     break;
 439                 list.add(acc);
 440             }
 441             throwIfCancelled();
 442
 443             Logger.warn("accounts",
 444                     String.format(Locale.US, "Got %d accounts using protocol %s", list.size(),
 445                             version.getDescription()));
 446         }
 447
 448         return list;
 449     }
 450     private List<LedgerTransaction> retrieveTransactionList()
 451             throws ParseException, HTTPException, IOException, ApiNotSupportedException {
 452         final API apiVersion = API.valueOf(profile.getApiVersion());
 453         if (apiVersion.equals(API.auto)) {
 454             return retrieveTransactionListAnyVersion();
 455         }
 456         else if (apiVersion.equals(API.html)) {
 457             Logger.debug("json",
 458                     "Declining using JSON API for /accounts with configured legacy API version");
 459             return null;
 460         }
 461         else {
 462             return retrieveTransactionListForVersion(apiVersion);
 463         }
 464
 465     }
 466     private List<LedgerTransaction> retrieveTransactionListAnyVersion()
 467             throws ApiNotSupportedException {
 468         for (API ver : API.allVersions) {
 469             try {
 470                 return retrieveTransactionListForVersion(ver);
 471             }
 472             catch (Exception e) {
 473                 Logger.debug("json", String.format(Locale.US,
 474                         "Error during transaction list retrieval using API %s",
 475                         ver.getDescription()), e);
 476             }
 477
 478         }
 479
 480         throw new ApiNotSupportedException();
 481     }
 482     private List<LedgerTransaction> retrieveTransactionListForVersion(API apiVersion)
 483             throws IOException, ParseException, HTTPException {
 484         Progress progress = new Progress();
 485         progress.setTotal(expectedPostingsCount);
 486
 487         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
 488         http.setAllowUserInteraction(false);
 489         publishProgress(progress);
 490         switch (http.getResponseCode()) {
 491             case 200:
 492                 break;
 493             case 404:
 494                 return null;
 495             default:
 496                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
 497         }
 498         ArrayList<LedgerTransaction> trList = new ArrayList<>();
 499         try (InputStream resp = http.getInputStream()) {
 500             throwIfCancelled();
 501
 502             TransactionListParser parser = TransactionListParser.forApiVersion(apiVersion, resp);
 503
 504             int processedPostings = 0;
 505
 506             while (true) {
 507                 throwIfCancelled();
 508                 LedgerTransaction transaction = parser.nextTransaction();
 509                 throwIfCancelled();
 510                 if (transaction == null)
 511                     break;
 512
 513                 trList.add(transaction);
 514
 515                 progress.setProgress(processedPostings += transaction.getAccounts()
 516                                                                      .size());
 517 //                Logger.debug("trParser",
 518 //                        String.format(Locale.US, "Parsed transaction %d - %s", transaction
 519 //                        .getId(),
 520 //                                transaction.getDescription()));
 521 //                for (LedgerTransactionAccount acc : transaction.getAccounts()) {
 522 //                    Logger.debug("trParser",
 523 //                            String.format(Locale.US, "  %s", acc.getAccountName()));
 524 //                }
 525                 publishProgress(progress);
 526             }
 527
 528             throwIfCancelled();
 529
 530             Logger.warn("transactions",
 531                     String.format(Locale.US, "Got %d transactions using protocol %s", trList.size(),
 532                             apiVersion.getDescription()));
 533         }
 534
 535         // json interface returns transactions in file order and the rest of the machinery
 536         // expects them in reverse chronological order
 537         Collections.sort(trList, (o1, o2) -> {
 538             int res = o2.getDate()
 539                         .compareTo(o1.getDate());
 540             if (res != 0)
 541                 return res;
 542             return Long.compare(o2.getLedgerId(), o1.getLedgerId());
 543         });
 544         return trList;
 545     }
 546
 547     @SuppressLint("DefaultLocale")
 548     @Override
 549     public void run() {
 550         Data.backgroundTaskStarted();
 551         List<LedgerAccount> accounts;
 552         List<LedgerTransaction> transactions;
 553         try {
 554             accounts = retrieveAccountList();
 555             // accounts is null in API-version auto-detection and means
 556             // requesting 'html' API version via the JSON classes
 557             // this can't work, and the null results in the legacy code below
 558             // being called
 559             if (accounts == null)
 560                 transactions = null;
 561             else
 562                 transactions = retrieveTransactionList();
 563
 564             if (accounts == null || transactions == null) {
 565                 accounts = new ArrayList<>();
 566                 transactions = new ArrayList<>();
 567                 retrieveTransactionListLegacy(accounts, transactions);
 568             }
 569
 570             new AccountAndTransactionListSaver(accounts, transactions).start();
 571
 572             Data.lastUpdateDate.postValue(new Date());
 573
 574             finish(new Result(null));
 575         }
 576         catch (MalformedURLException e) {
 577             e.printStackTrace();
 578             finish(new Result("Invalid server URL"));
 579         }
 580         catch (HTTPException e) {
 581             e.printStackTrace();
 582             finish(new Result(
 583                     String.format("HTTP error %d: %s", e.getResponseCode(), e.getMessage())));
 584         }
 585         catch (IOException e) {
 586             e.printStackTrace();
 587             finish(new Result(e.getLocalizedMessage()));
 588         }
 589         catch (RuntimeJsonMappingException e) {
 590             e.printStackTrace();
 591             finish(new Result(Result.ERR_JSON_PARSER_ERROR));
 592         }
 593         catch (ParseException e) {
 594             e.printStackTrace();
 595             finish(new Result("Network error"));
 596         }
 597         catch (OperationCanceledException e) {
 598             Logger.debug("RTT", "Retrieval was cancelled", e);
 599             finish(new Result(null));
 600         }
 601         catch (ApiNotSupportedException e) {
 602             e.printStackTrace();
 603             finish(new Result("Server version not supported"));
 604         }
 605         finally {
 606             Data.backgroundTaskFinished();
 607         }
 608     }
 609     public void throwIfCancelled() {
 610         if (isInterrupted())
 611             throw new OperationCanceledException(null);
 612     }
 613     private enum ParserState {
 614         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
 615         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
 616     }
 617
 618     public enum ProgressState {STARTING, RUNNING, FINISHED}
 619
 620     public static class Progress {
 621         private int progress;
 622         private int total;
 623         private ProgressState state = ProgressState.RUNNING;
 624         private String error = null;
 625         private boolean indeterminate;
 626         Progress() {
 627             indeterminate = true;
 628         }
 629         Progress(int progress, int total) {
 630             this.indeterminate = false;
 631             this.progress = progress;
 632             this.total = total;
 633         }
 634         public static Progress indeterminate() {
 635             return new Progress();
 636         }
 637         public static Progress finished(String error) {
 638             Progress p = new Progress();
 639             p.setState(ProgressState.FINISHED);
 640             p.setError(error);
 641             return p;
 642         }
 643         public int getProgress() {
 644             ensureState(ProgressState.RUNNING);
 645             return progress;
 646         }
 647         protected void setProgress(int progress) {
 648             this.progress = progress;
 649             this.state = ProgressState.RUNNING;
 650         }
 651         public int getTotal() {
 652             ensureState(ProgressState.RUNNING);
 653             return total;
 654         }
 655         protected void setTotal(int total) {
 656             this.total = total;
 657             state = ProgressState.RUNNING;
 658             indeterminate = total == -1;
 659         }
 660         private void ensureState(ProgressState wanted) {
 661             if (state != wanted)
 662                 throw new IllegalStateException(
 663                         String.format("Bad state: %s, expected %s", state, wanted));
 664         }
 665         public ProgressState getState() {
 666             return state;
 667         }
 668         public void setState(ProgressState state) {
 669             this.state = state;
 670         }
 671         public String getError() {
 672             ensureState(ProgressState.FINISHED);
 673             return error;
 674         }
 675         public void setError(String error) {
 676             this.error = error;
 677             state = ProgressState.FINISHED;
 678         }
 679         public boolean isIndeterminate() {
 680             return indeterminate;
 681         }
 682         public void setIndeterminate(boolean indeterminate) {
 683             this.indeterminate = indeterminate;
 684         }
 685     }
 686
 687     private static class TransactionParserException extends IllegalStateException {
 688         TransactionParserException(String message) {
 689             super(message);
 690         }
 691     }
 692
 693     public static class Result {
 694         public static String ERR_JSON_PARSER_ERROR = "err_json_parser";
 695         public String error;
 696         public List<LedgerAccount> accounts;
 697         public List<LedgerTransaction> transactions;
 698         Result(String error) {
 699             this.error = error;
 700         }
 701         Result(List<LedgerAccount> accounts, List<LedgerTransaction> transactions) {
 702             this.accounts = accounts;
 703             this.transactions = transactions;
 704         }
 705     }
 706
 707     private class AccountAndTransactionListSaver extends Thread {
 708         private final List<LedgerAccount> accounts;
 709         private final List<LedgerTransaction> transactions;
 710         public AccountAndTransactionListSaver(List<LedgerAccount> accounts,
 711                                               List<LedgerTransaction> transactions) {
 712             this.accounts = accounts;
 713             this.transactions = transactions;
 714         }
 715         @Override
 716         public void run() {
 717             AccountDAO accDao = DB.get()
 718                                   .getAccountDAO();
 719             TransactionDAO trDao = DB.get()
 720                                      .getTransactionDAO();
 721
 722             Logger.debug(TAG, "Preparing account list");
 723             final List<AccountWithAmounts> list = new ArrayList<>();
 724             for (LedgerAccount acc : accounts) {
 725                 final AccountWithAmounts a = acc.toDBOWithAmounts();
 726                 Account existing = accDao.getByNameSync(profile.getId(), acc.getName());
 727                 if (existing != null) {
 728                     a.account.setExpanded(existing.isExpanded());
 729                     a.account.setAmountsExpanded(existing.isAmountsExpanded());
 730                     a.account.setId(existing.getId()); // not strictly needed, but since we have it
 731                     // anyway...
 732                 }
 733
 734                 list.add(a);
 735             }
 736             Logger.debug(TAG, "Account list prepared. Storing");
 737             accDao.storeAccountsSync(list, profile.getId());
 738             Logger.debug(TAG, "Account list stored");
 739
 740             Logger.debug(TAG, "Preparing transaction list");
 741             final List<TransactionWithAccounts> tranList = new ArrayList<>();
 742
 743             for (LedgerTransaction tr : transactions)
 744                 tranList.add(tr.toDBO());
 745
 746             Logger.debug(TAG, "Storing transaction list");
 747             trDao.storeTransactionsSync(tranList, profile.getId());
 748
 749             Logger.debug(TAG, "Transactions stored");
 750
 751             DB.get()
 752               .getOptionDAO()
 753               .insertSync(new Option(profile.getId(), Option.OPT_LAST_SCRAPE,
 754                       String.valueOf((new Date()).getTime())));
 755         }
 756     }
 757 }