]> git.ktnx.net Git - mobile-ledger.git/blob - app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java
Room-based profile management
[mobile-ledger.git] / app / src / main / java / net / ktnx / mobileledger / async / RetrieveTransactionsTask.java
1 /*
2  * Copyright © 2021 Damyan Ivanov.
3  * This file is part of MoLe.
4  * MoLe is free software: you can distribute it and/or modify it
5  * under the term of the GNU General Public License as published by
6  * the Free Software Foundation, either version 3 of the License, or
7  * (at your opinion), any later version.
8  *
9  * MoLe is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  * GNU General Public License terms for details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
16  */
17
18 package net.ktnx.mobileledger.async;
19
20 import android.annotation.SuppressLint;
21 import android.database.sqlite.SQLiteDatabase;
22 import android.os.AsyncTask;
23 import android.os.OperationCanceledException;
24
25 import androidx.annotation.NonNull;
26 import androidx.room.Transaction;
27
28 import com.fasterxml.jackson.databind.RuntimeJsonMappingException;
29
30 import net.ktnx.mobileledger.App;
31 import net.ktnx.mobileledger.dao.AccountDAO;
32 import net.ktnx.mobileledger.dao.AccountValueDAO;
33 import net.ktnx.mobileledger.dao.TransactionAccountDAO;
34 import net.ktnx.mobileledger.dao.TransactionDAO;
35 import net.ktnx.mobileledger.db.Account;
36 import net.ktnx.mobileledger.db.AccountWithAmounts;
37 import net.ktnx.mobileledger.db.DB;
38 import net.ktnx.mobileledger.db.Option;
39 import net.ktnx.mobileledger.db.Profile;
40 import net.ktnx.mobileledger.db.TransactionAccount;
41 import net.ktnx.mobileledger.db.TransactionWithAccounts;
42 import net.ktnx.mobileledger.err.HTTPException;
43 import net.ktnx.mobileledger.json.API;
44 import net.ktnx.mobileledger.json.AccountListParser;
45 import net.ktnx.mobileledger.json.ApiNotSupportedException;
46 import net.ktnx.mobileledger.json.TransactionListParser;
47 import net.ktnx.mobileledger.model.Data;
48 import net.ktnx.mobileledger.model.LedgerAccount;
49 import net.ktnx.mobileledger.model.LedgerTransaction;
50 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
51 import net.ktnx.mobileledger.ui.MainModel;
52 import net.ktnx.mobileledger.utils.Logger;
53 import net.ktnx.mobileledger.utils.MLDB;
54 import net.ktnx.mobileledger.utils.NetworkUtil;
55
56 import java.io.BufferedReader;
57 import java.io.IOException;
58 import java.io.InputStream;
59 import java.io.InputStreamReader;
60 import java.net.HttpURLConnection;
61 import java.net.MalformedURLException;
62 import java.net.URLDecoder;
63 import java.nio.charset.StandardCharsets;
64 import java.text.ParseException;
65 import java.util.ArrayList;
66 import java.util.Collections;
67 import java.util.Date;
68 import java.util.HashMap;
69 import java.util.List;
70 import java.util.Locale;
71 import java.util.Objects;
72 import java.util.regex.Matcher;
73 import java.util.regex.Pattern;
74
75
76 public class RetrieveTransactionsTask extends
77         AsyncTask<Void, RetrieveTransactionsTask.Progress, RetrieveTransactionsTask.Result> {
78     private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
79     private static final Pattern reComment = Pattern.compile("^\\s*;");
80     private static final Pattern reTransactionStart = Pattern.compile(
81             "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
82             "\"[^\"]*>([\\d.-]+)</td>");
83     private static final Pattern reTransactionDescription =
84             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
85     private static final Pattern reTransactionDetails = Pattern.compile(
86             "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
87             "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
88     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
89     private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
90     private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
91     // %3A is '='
92     private final Pattern reAccountName =
93             Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
94     private final Pattern reAccountValue = Pattern.compile(
95             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
96     private final MainModel mainModel;
97     private final Profile profile;
98     private int expectedPostingsCount = -1;
99     public RetrieveTransactionsTask(@NonNull MainModel mainModel, @NonNull Profile profile) {
100         this.mainModel = mainModel;
101         this.profile = profile;
102     }
103     private static void L(String msg) {
104         //debug("transaction-parser", msg);
105     }
106     static LedgerTransactionAccount parseTransactionAccountLine(String line) {
107         Matcher m = reTransactionDetails.matcher(line);
108         if (m.find()) {
109             String postingStatus = m.group(1);
110             String acc_name = m.group(2);
111             String currencyPre = m.group(3);
112             String amount = Objects.requireNonNull(m.group(4));
113             String currencyPost = m.group(5);
114
115             String currency = null;
116             if ((currencyPre != null) && (currencyPre.length() > 0)) {
117                 if ((currencyPost != null) && (currencyPost.length() > 0))
118                     return null;
119                 currency = currencyPre;
120             }
121             else if ((currencyPost != null) && (currencyPost.length() > 0)) {
122                 currency = currencyPost;
123             }
124
125             amount = amount.replace(',', '.');
126
127             return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
128         }
129         else {
130             return null;
131         }
132     }
133     @Override
134     protected void onProgressUpdate(Progress... values) {
135         super.onProgressUpdate(values);
136         Data.backgroundTaskProgress.postValue(values[0]);
137     }
138     @Override
139     protected void onPostExecute(Result result) {
140         super.onPostExecute(result);
141         Progress progress = new Progress();
142         progress.setState(ProgressState.FINISHED);
143         progress.setError(result.error);
144         onProgressUpdate(progress);
145     }
146     @Override
147     protected void onCancelled() {
148         super.onCancelled();
149         Progress progress = new Progress();
150         progress.setState(ProgressState.FINISHED);
151         onProgressUpdate(progress);
152     }
153     private void retrieveTransactionListLegacy(List<LedgerAccount> accounts,
154                                                List<LedgerTransaction> transactions)
155             throws IOException, HTTPException {
156         Progress progress = Progress.indeterminate();
157         progress.setState(ProgressState.RUNNING);
158         progress.setTotal(expectedPostingsCount);
159         int maxTransactionId = -1;
160         HashMap<String, LedgerAccount> map = new HashMap<>();
161         LedgerAccount lastAccount = null;
162         ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
163
164         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
165         http.setAllowUserInteraction(false);
166         publishProgress(progress);
167         if (http.getResponseCode() != 200)
168             throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
169
170         try (InputStream resp = http.getInputStream()) {
171             if (http.getResponseCode() != 200)
172                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
173
174             int matchedTransactionsCount = 0;
175
176             ParserState state = ParserState.EXPECTING_ACCOUNT;
177             String line;
178             BufferedReader buf =
179                     new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
180
181             int processedTransactionCount = 0;
182             int transactionId = 0;
183             LedgerTransaction transaction = null;
184             LINES:
185             while ((line = buf.readLine()) != null) {
186                 throwIfCancelled();
187                 Matcher m;
188                 m = reComment.matcher(line);
189                 if (m.find()) {
190                     // TODO: comments are ignored for now
191 //                            Log.v("transaction-parser", "Ignoring comment");
192                     continue;
193                 }
194                 //L(String.format("State is %d", updating));
195                 switch (state) {
196                     case EXPECTING_ACCOUNT:
197                         if (line.equals("<h2>General Journal</h2>")) {
198                             state = ParserState.EXPECTING_TRANSACTION;
199                             L("→ expecting transaction");
200                             continue;
201                         }
202                         m = reAccountName.matcher(line);
203                         if (m.find()) {
204                             String acct_encoded = m.group(1);
205                             String accName = URLDecoder.decode(acct_encoded, "UTF-8");
206                             accName = accName.replace("\"", "");
207                             L(String.format("found account: %s", accName));
208
209                             lastAccount = map.get(accName);
210                             if (lastAccount != null) {
211                                 L(String.format("ignoring duplicate account '%s'", accName));
212                                 continue;
213                             }
214                             String parentAccountName = LedgerAccount.extractParentName(accName);
215                             LedgerAccount parentAccount;
216                             if (parentAccountName != null) {
217                                 parentAccount = ensureAccountExists(parentAccountName, map,
218                                         syntheticAccounts);
219                             }
220                             else {
221                                 parentAccount = null;
222                             }
223                             lastAccount = new LedgerAccount(accName, parentAccount);
224
225                             accounts.add(lastAccount);
226                             map.put(accName, lastAccount);
227
228                             state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
229                             L("→ expecting account amount");
230                         }
231                         break;
232
233                     case EXPECTING_ACCOUNT_AMOUNT:
234                         m = reAccountValue.matcher(line);
235                         boolean match_found = false;
236                         while (m.find()) {
237                             throwIfCancelled();
238
239                             match_found = true;
240                             String value = Objects.requireNonNull(m.group(1));
241                             String currency = m.group(2);
242                             if (currency == null)
243                                 currency = "";
244
245                             {
246                                 Matcher tmpM = reDecimalComma.matcher(value);
247                                 if (tmpM.find()) {
248                                     value = value.replace(".", "");
249                                     value = value.replace(',', '.');
250                                 }
251
252                                 tmpM = reDecimalPoint.matcher(value);
253                                 if (tmpM.find()) {
254                                     value = value.replace(",", "");
255                                     value = value.replace(" ", "");
256                                 }
257                             }
258                             L("curr=" + currency + ", value=" + value);
259                             final float val = Float.parseFloat(value);
260                             lastAccount.addAmount(val, currency);
261                             for (LedgerAccount syn : syntheticAccounts) {
262                                 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
263                                         currency, val, syn.getName()));
264                                 syn.addAmount(val, currency);
265                             }
266                         }
267
268                         if (match_found) {
269                             syntheticAccounts.clear();
270                             state = ParserState.EXPECTING_ACCOUNT;
271                             L("→ expecting account");
272                         }
273
274                         break;
275
276                     case EXPECTING_TRANSACTION:
277                         if (!line.isEmpty() && (line.charAt(0) == ' '))
278                             continue;
279                         m = reTransactionStart.matcher(line);
280                         if (m.find()) {
281                             transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
282                             state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
283                             L(String.format(Locale.ENGLISH,
284                                     "found transaction %d → expecting description", transactionId));
285                             progress.setProgress(++processedTransactionCount);
286                             if (maxTransactionId < transactionId)
287                                 maxTransactionId = transactionId;
288                             if ((progress.isIndeterminate()) ||
289                                 (progress.getTotal() < transactionId))
290                                 progress.setTotal(transactionId);
291                             publishProgress(progress);
292                         }
293                         m = reEnd.matcher(line);
294                         if (m.find()) {
295                             L("--- transaction value complete ---");
296                             break LINES;
297                         }
298                         break;
299
300                     case EXPECTING_TRANSACTION_DESCRIPTION:
301                         if (!line.isEmpty() && (line.charAt(0) == ' '))
302                             continue;
303                         m = reTransactionDescription.matcher(line);
304                         if (m.find()) {
305                             if (transactionId == 0)
306                                 throw new TransactionParserException(
307                                         "Transaction Id is 0 while expecting description");
308
309                             String date = Objects.requireNonNull(m.group(1));
310                             try {
311                                 int equalsIndex = date.indexOf('=');
312                                 if (equalsIndex >= 0)
313                                     date = date.substring(equalsIndex + 1);
314                                 transaction =
315                                         new LedgerTransaction(transactionId, date, m.group(2));
316                             }
317                             catch (ParseException e) {
318                                 throw new TransactionParserException(
319                                         String.format("Error parsing date '%s'", date));
320                             }
321                             state = ParserState.EXPECTING_TRANSACTION_DETAILS;
322                             L(String.format(Locale.ENGLISH,
323                                     "transaction %d created for %s (%s) →" + " expecting details",
324                                     transactionId, date, m.group(2)));
325                         }
326                         break;
327
328                     case EXPECTING_TRANSACTION_DETAILS:
329                         if (line.isEmpty()) {
330                             // transaction data collected
331
332                             transaction.finishLoading();
333                             transactions.add(transaction);
334
335                             state = ParserState.EXPECTING_TRANSACTION;
336                             L(String.format("transaction %s parsed → expecting transaction",
337                                     transaction.getLedgerId()));
338
339 // sounds like a good idea, but transaction-1 may not be the first one chronologically
340 // for example, when you add the initial seeding transaction after entering some others
341 //                                            if (transactionId == 1) {
342 //                                                L("This was the initial transaction.
343 //                                                Terminating " +
344 //                                                  "parser");
345 //                                                break LINES;
346 //                                            }
347                         }
348                         else {
349                             LedgerTransactionAccount lta = parseTransactionAccountLine(line);
350                             if (lta != null) {
351                                 transaction.addAccount(lta);
352                                 L(String.format(Locale.ENGLISH, "%d: %s = %s",
353                                         transaction.getLedgerId(), lta.getAccountName(),
354                                         lta.getAmount()));
355                             }
356                             else
357                                 throw new IllegalStateException(
358                                         String.format("Can't parse transaction %d details: %s",
359                                                 transactionId, line));
360                         }
361                         break;
362                     default:
363                         throw new RuntimeException(
364                                 String.format("Unknown parser updating %s", state.name()));
365                 }
366             }
367
368             throwIfCancelled();
369         }
370     }
371     @NonNull
372     public LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
373                                              ArrayList<LedgerAccount> createdAccounts) {
374         LedgerAccount acc = map.get(accountName);
375
376         if (acc != null)
377             return acc;
378
379         String parentName = LedgerAccount.extractParentName(accountName);
380         LedgerAccount parentAccount;
381         if (parentName != null) {
382             parentAccount = ensureAccountExists(parentName, map, createdAccounts);
383         }
384         else {
385             parentAccount = null;
386         }
387
388         acc = new LedgerAccount(accountName, parentAccount);
389         createdAccounts.add(acc);
390         return acc;
391     }
392     public void addNumberOfPostings(int number) {
393         expectedPostingsCount += number;
394     }
395     private List<LedgerAccount> retrieveAccountList()
396             throws IOException, HTTPException, ApiNotSupportedException {
397         final API apiVersion = API.valueOf(profile.getApiVersion());
398         if (apiVersion.equals(API.auto)) {
399             return retrieveAccountListAnyVersion();
400         }
401         else if (apiVersion.equals(API.html)) {
402             Logger.debug("json",
403                     "Declining using JSON API for /accounts with configured legacy API version");
404             return null;
405         }
406         else {
407             return retrieveAccountListForVersion(apiVersion);
408         }
409     }
410     private List<LedgerAccount> retrieveAccountListAnyVersion()
411             throws HTTPException, ApiNotSupportedException {
412         for (API ver : API.allVersions) {
413             try {
414                 return retrieveAccountListForVersion(ver);
415             }
416             catch (Exception e) {
417                 Logger.debug("json",
418                         String.format(Locale.US, "Error during account list retrieval using API %s",
419                                 ver.getDescription()));
420             }
421
422             throw new ApiNotSupportedException();
423         }
424
425         throw new RuntimeException("This should never be reached");
426     }
427     private List<LedgerAccount> retrieveAccountListForVersion(API version)
428             throws IOException, HTTPException {
429         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
430         http.setAllowUserInteraction(false);
431         switch (http.getResponseCode()) {
432             case 200:
433                 break;
434             case 404:
435                 return null;
436             default:
437                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
438         }
439         publishProgress(Progress.indeterminate());
440         SQLiteDatabase db = App.getDatabase();
441         ArrayList<LedgerAccount> list = new ArrayList<>();
442         HashMap<String, LedgerAccount> map = new HashMap<>();
443         throwIfCancelled();
444         try (InputStream resp = http.getInputStream()) {
445             throwIfCancelled();
446             if (http.getResponseCode() != 200)
447                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
448
449             AccountListParser parser = AccountListParser.forApiVersion(version, resp);
450             expectedPostingsCount = 0;
451
452             while (true) {
453                 throwIfCancelled();
454                 LedgerAccount acc = parser.nextAccount(this, map);
455                 if (acc == null)
456                     break;
457                 list.add(acc);
458             }
459             throwIfCancelled();
460         }
461
462         return list;
463     }
464     private List<LedgerTransaction> retrieveTransactionList()
465             throws ParseException, HTTPException, IOException, ApiNotSupportedException {
466         final API apiVersion = API.valueOf(profile.getApiVersion());
467         if (apiVersion.equals(API.auto)) {
468             return retrieveTransactionListAnyVersion();
469         }
470         else if (apiVersion.equals(API.html)) {
471             Logger.debug("json",
472                     "Declining using JSON API for /accounts with configured legacy API version");
473             return null;
474         }
475         else {
476             return retrieveTransactionListForVersion(apiVersion);
477         }
478
479     }
480     private List<LedgerTransaction> retrieveTransactionListAnyVersion()
481             throws ApiNotSupportedException {
482         for (API ver : API.allVersions) {
483             try {
484                 return retrieveTransactionListForVersion(ver);
485             }
486             catch (Exception | HTTPException e) {
487                 Logger.debug("json",
488                         String.format(Locale.US, "Error during account list retrieval using API %s",
489                                 ver.getDescription()));
490             }
491
492             throw new ApiNotSupportedException();
493         }
494
495         throw new RuntimeException("This should never be reached");
496     }
497     private List<LedgerTransaction> retrieveTransactionListForVersion(API apiVersion)
498             throws IOException, ParseException, HTTPException {
499         Progress progress = new Progress();
500         progress.setTotal(expectedPostingsCount);
501
502         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
503         http.setAllowUserInteraction(false);
504         publishProgress(progress);
505         switch (http.getResponseCode()) {
506             case 200:
507                 break;
508             case 404:
509                 return null;
510             default:
511                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
512         }
513         ArrayList<LedgerTransaction> trList = new ArrayList<>();
514         try (InputStream resp = http.getInputStream()) {
515             throwIfCancelled();
516
517             TransactionListParser parser = TransactionListParser.forApiVersion(apiVersion, resp);
518
519             int processedPostings = 0;
520
521             while (true) {
522                 throwIfCancelled();
523                 LedgerTransaction transaction = parser.nextTransaction();
524                 throwIfCancelled();
525                 if (transaction == null)
526                     break;
527
528                 trList.add(transaction);
529
530                 progress.setProgress(processedPostings += transaction.getAccounts()
531                                                                      .size());
532 //                Logger.debug("trParser",
533 //                        String.format(Locale.US, "Parsed transaction %d - %s", transaction
534 //                        .getId(),
535 //                                transaction.getDescription()));
536 //                for (LedgerTransactionAccount acc : transaction.getAccounts()) {
537 //                    Logger.debug("trParser",
538 //                            String.format(Locale.US, "  %s", acc.getAccountName()));
539 //                }
540                 publishProgress(progress);
541             }
542
543             throwIfCancelled();
544         }
545
546         // json interface returns transactions if file order and the rest of the machinery
547         // expects them in reverse chronological order
548         Collections.sort(trList, (o1, o2) -> {
549             int res = o2.getDate()
550                         .compareTo(o1.getDate());
551             if (res != 0)
552                 return res;
553             return Long.compare(o2.getLedgerId(), o1.getLedgerId());
554         });
555         return trList;
556     }
557
558     @SuppressLint("DefaultLocale")
559     @Override
560     protected Result doInBackground(Void... params) {
561         Data.backgroundTaskStarted();
562         List<LedgerAccount> accounts;
563         List<LedgerTransaction> transactions;
564         try {
565             accounts = retrieveAccountList();
566             // accounts is null in API-version auto-detection and means
567             // requesting 'html' API version via the JSON classes
568             // this can't work, and the null results in the legacy code below
569             // being called
570             if (accounts == null)
571                 transactions = null;
572             else
573                 transactions = retrieveTransactionList();
574
575             if (accounts == null || transactions == null) {
576                 accounts = new ArrayList<>();
577                 transactions = new ArrayList<>();
578                 retrieveTransactionListLegacy(accounts, transactions);
579             }
580
581             storeAccountsAndTransactions(accounts, transactions);
582
583             mainModel.updateDisplayedTransactionsFromWeb(transactions);
584
585             return new Result(accounts, transactions);
586         }
587         catch (MalformedURLException e) {
588             e.printStackTrace();
589             return new Result("Invalid server URL");
590         }
591         catch (HTTPException e) {
592             e.printStackTrace();
593             return new Result(String.format("HTTP error %d: %s", e.getResponseCode(),
594                     e.getResponseMessage()));
595         }
596         catch (IOException e) {
597             e.printStackTrace();
598             return new Result(e.getLocalizedMessage());
599         }
600         catch (RuntimeJsonMappingException e) {
601             e.printStackTrace();
602             return new Result(Result.ERR_JSON_PARSER_ERROR);
603         }
604         catch (ParseException e) {
605             e.printStackTrace();
606             return new Result("Network error");
607         }
608         catch (OperationCanceledException e) {
609             e.printStackTrace();
610             return new Result("Operation cancelled");
611         }
612         catch (ApiNotSupportedException e) {
613             e.printStackTrace();
614             return new Result("Server version not supported");
615         }
616         finally {
617             Data.backgroundTaskFinished();
618         }
619     }
620     @Transaction
621     private void storeAccountsAndTransactions(List<LedgerAccount> accounts,
622                                               List<LedgerTransaction> transactions) {
623         AccountDAO accDao = DB.get()
624                               .getAccountDAO();
625         TransactionDAO trDao = DB.get()
626                                  .getTransactionDAO();
627         TransactionAccountDAO trAccDao = DB.get()
628                                            .getTransactionAccountDAO();
629         AccountValueDAO valDao = DB.get()
630                                    .getAccountValueDAO();
631
632         final List<AccountWithAmounts> list = new ArrayList<>();
633         for (LedgerAccount acc : accounts) {
634             final AccountWithAmounts a = acc.toDBOWithAmounts();
635             Account existing = accDao.getByNameSync(profile.getId(), acc.getName());
636             if (existing != null) {
637                 a.account.setExpanded(existing.isExpanded());
638                 a.account.setAmountsExpanded(existing.isAmountsExpanded());
639                 a.account.setId(
640                         existing.getId()); // not strictly needed, but since we have it anyway...
641             }
642
643             list.add(a);
644         }
645         accDao.storeAccountsSync(list, profile.getId());
646
647         long trGen = trDao.getGenerationSync(profile.getId());
648         for (LedgerTransaction tr : transactions) {
649             TransactionWithAccounts tran = tr.toDBO();
650             tran.transaction.setGeneration(trGen);
651             tran.transaction.setProfileId(profile.getId());
652
653             tran.transaction.setId(trDao.insertSync(tran.transaction));
654
655             for (TransactionAccount trAcc : tran.accounts) {
656                 trAcc.setGeneration(trGen);
657                 trAcc.setTransactionId(tran.transaction.getId());
658                 trAcc.setId(trAccDao.insertSync(trAcc));
659             }
660         }
661
662         trDao.purgeOldTransactionsSync(profile.getId(), trGen);
663
664         DB.get()
665           .getOptionDAO()
666           .insertSync(new Option(profile.getId(), MLDB.OPT_LAST_SCRAPE,
667                   String.valueOf((new Date()).getTime())));
668     }
669     public void throwIfCancelled() {
670         if (isCancelled())
671             throw new OperationCanceledException(null);
672     }
673     private enum ParserState {
674         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
675         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
676     }
677
678     public enum ProgressState {STARTING, RUNNING, FINISHED}
679
680     public static class Progress {
681         private int progress;
682         private int total;
683         private ProgressState state = ProgressState.RUNNING;
684         private String error = null;
685         private boolean indeterminate;
686         Progress() {
687             indeterminate = true;
688         }
689         Progress(int progress, int total) {
690             this.indeterminate = false;
691             this.progress = progress;
692             this.total = total;
693         }
694         public static Progress indeterminate() {
695             return new Progress();
696         }
697         public static Progress finished(String error) {
698             Progress p = new Progress();
699             p.setState(ProgressState.FINISHED);
700             p.setError(error);
701             return p;
702         }
703         public int getProgress() {
704             ensureState(ProgressState.RUNNING);
705             return progress;
706         }
707         protected void setProgress(int progress) {
708             this.progress = progress;
709             this.state = ProgressState.RUNNING;
710         }
711         public int getTotal() {
712             ensureState(ProgressState.RUNNING);
713             return total;
714         }
715         protected void setTotal(int total) {
716             this.total = total;
717             state = ProgressState.RUNNING;
718             indeterminate = total == -1;
719         }
720         private void ensureState(ProgressState wanted) {
721             if (state != wanted)
722                 throw new IllegalStateException(
723                         String.format("Bad state: %s, expected %s", state, wanted));
724         }
725         public ProgressState getState() {
726             return state;
727         }
728         public void setState(ProgressState state) {
729             this.state = state;
730         }
731         public String getError() {
732             ensureState(ProgressState.FINISHED);
733             return error;
734         }
735         public void setError(String error) {
736             this.error = error;
737             state = ProgressState.FINISHED;
738         }
739         public boolean isIndeterminate() {
740             return indeterminate;
741         }
742         public void setIndeterminate(boolean indeterminate) {
743             this.indeterminate = indeterminate;
744         }
745     }
746
747     private static class TransactionParserException extends IllegalStateException {
748         TransactionParserException(String message) {
749             super(message);
750         }
751     }
752
753     public static class Result {
754         public static String ERR_JSON_PARSER_ERROR = "err_json_parser";
755         public String error;
756         public List<LedgerAccount> accounts;
757         public List<LedgerTransaction> transactions;
758         Result(String error) {
759             this.error = error;
760         }
761         Result(List<LedgerAccount> accounts, List<LedgerTransaction> transactions) {
762             this.accounts = accounts;
763             this.transactions = transactions;
764         }
765     }
766 }