]> git.ktnx.net Git - mobile-ledger-staging.git/blob - app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java
a step towards API-independent JSON parsing of accounts
[mobile-ledger-staging.git] / app / src / main / java / net / ktnx / mobileledger / async / RetrieveTransactionsTask.java
1 /*
2  * Copyright © 2020 Damyan Ivanov.
3  * This file is part of MoLe.
4  * MoLe is free software: you can distribute it and/or modify it
5  * under the term of the GNU General Public License as published by
6  * the Free Software Foundation, either version 3 of the License, or
7  * (at your opinion), any later version.
8  *
9  * MoLe is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  * GNU General Public License terms for details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
16  */
17
18 package net.ktnx.mobileledger.async;
19
20 import android.annotation.SuppressLint;
21 import android.database.sqlite.SQLiteDatabase;
22 import android.os.AsyncTask;
23 import android.os.OperationCanceledException;
24
25 import androidx.annotation.NonNull;
26
27 import net.ktnx.mobileledger.App;
28 import net.ktnx.mobileledger.err.HTTPException;
29 import net.ktnx.mobileledger.json.v1_15.AccountListParser;
30 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerTransaction;
31 import net.ktnx.mobileledger.json.v1_15.TransactionListParser;
32 import net.ktnx.mobileledger.model.Data;
33 import net.ktnx.mobileledger.model.LedgerAccount;
34 import net.ktnx.mobileledger.model.LedgerTransaction;
35 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
36 import net.ktnx.mobileledger.model.MobileLedgerProfile;
37 import net.ktnx.mobileledger.ui.MainModel;
38 import net.ktnx.mobileledger.utils.NetworkUtil;
39
40 import java.io.BufferedReader;
41 import java.io.IOException;
42 import java.io.InputStream;
43 import java.io.InputStreamReader;
44 import java.net.HttpURLConnection;
45 import java.net.MalformedURLException;
46 import java.net.URLDecoder;
47 import java.nio.charset.StandardCharsets;
48 import java.text.ParseException;
49 import java.util.ArrayList;
50 import java.util.Collections;
51 import java.util.HashMap;
52 import java.util.List;
53 import java.util.Locale;
54 import java.util.Objects;
55 import java.util.regex.Matcher;
56 import java.util.regex.Pattern;
57
58
59 public class RetrieveTransactionsTask extends
60         AsyncTask<Void, RetrieveTransactionsTask.Progress, RetrieveTransactionsTask.Result> {
61     private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
62     private static final Pattern reComment = Pattern.compile("^\\s*;");
63     private static final Pattern reTransactionStart = Pattern.compile(
64             "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
65             "\"[^\"]*>([\\d.-]+)</td>");
66     private static final Pattern reTransactionDescription =
67             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
68     private static final Pattern reTransactionDetails = Pattern.compile(
69             "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
70             "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
71     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
72     private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
73     private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
74     // %3A is '='
75     private final Pattern reAccountName =
76             Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
77     private final Pattern reAccountValue = Pattern.compile(
78             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
79     private final MainModel mainModel;
80     private final MobileLedgerProfile profile;
81     private final List<LedgerAccount> prevAccounts;
82     private int expectedPostingsCount = -1;
83     public RetrieveTransactionsTask(@NonNull MainModel mainModel,
84                                     @NonNull MobileLedgerProfile profile,
85                                     List<LedgerAccount> accounts) {
86         this.mainModel = mainModel;
87         this.profile = profile;
88         this.prevAccounts = accounts;
89     }
90     private static void L(String msg) {
91         //debug("transaction-parser", msg);
92     }
93     static LedgerTransactionAccount parseTransactionAccountLine(String line) {
94         Matcher m = reTransactionDetails.matcher(line);
95         if (m.find()) {
96             String postingStatus = m.group(1);
97             String acc_name = m.group(2);
98             String currencyPre = m.group(3);
99             String amount = Objects.requireNonNull(m.group(4));
100             String currencyPost = m.group(5);
101
102             String currency = null;
103             if ((currencyPre != null) && (currencyPre.length() > 0)) {
104                 if ((currencyPost != null) && (currencyPost.length() > 0))
105                     return null;
106                 currency = currencyPre;
107             }
108             else if ((currencyPost != null) && (currencyPost.length() > 0)) {
109                 currency = currencyPost;
110             }
111
112             amount = amount.replace(',', '.');
113
114             return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
115         }
116         else {
117             return null;
118         }
119     }
120     public MobileLedgerProfile getProfile() {
121         return profile;
122     }
123     @Override
124     protected void onProgressUpdate(Progress... values) {
125         super.onProgressUpdate(values);
126         Data.backgroundTaskProgress.postValue(values[0]);
127     }
128     @Override
129     protected void onPostExecute(Result result) {
130         super.onPostExecute(result);
131         Progress progress = new Progress();
132         progress.setState(ProgressState.FINISHED);
133         progress.setError(result.error);
134         onProgressUpdate(progress);
135     }
136     @Override
137     protected void onCancelled() {
138         super.onCancelled();
139         Progress progress = new Progress();
140         progress.setState(ProgressState.FINISHED);
141         onProgressUpdate(progress);
142     }
143     private void retrieveTransactionListLegacy(List<LedgerAccount> accounts,
144                                                List<LedgerTransaction> transactions)
145             throws IOException, HTTPException {
146         Progress progress = Progress.indeterminate();
147         progress.setState(ProgressState.RUNNING);
148         progress.setTotal(expectedPostingsCount);
149         int maxTransactionId = -1;
150         HashMap<String, LedgerAccount> map = new HashMap<>();
151         LedgerAccount lastAccount = null;
152         ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
153
154         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
155         http.setAllowUserInteraction(false);
156         publishProgress(progress);
157         if (http.getResponseCode() != 200)
158             throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
159
160         try (InputStream resp = http.getInputStream()) {
161             if (http.getResponseCode() != 200)
162                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
163
164             int matchedTransactionsCount = 0;
165
166             ParserState state = ParserState.EXPECTING_ACCOUNT;
167             String line;
168             BufferedReader buf =
169                     new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
170
171             int processedTransactionCount = 0;
172             int transactionId = 0;
173             LedgerTransaction transaction = null;
174             LINES:
175             while ((line = buf.readLine()) != null) {
176                 throwIfCancelled();
177                 Matcher m;
178                 m = reComment.matcher(line);
179                 if (m.find()) {
180                     // TODO: comments are ignored for now
181 //                            Log.v("transaction-parser", "Ignoring comment");
182                     continue;
183                 }
184                 //L(String.format("State is %d", updating));
185                 switch (state) {
186                     case EXPECTING_ACCOUNT:
187                         if (line.equals("<h2>General Journal</h2>")) {
188                             state = ParserState.EXPECTING_TRANSACTION;
189                             L("→ expecting transaction");
190                             continue;
191                         }
192                         m = reAccountName.matcher(line);
193                         if (m.find()) {
194                             String acct_encoded = m.group(1);
195                             String accName = URLDecoder.decode(acct_encoded, "UTF-8");
196                             accName = accName.replace("\"", "");
197                             L(String.format("found account: %s", accName));
198
199                             lastAccount = map.get(accName);
200                             if (lastAccount != null) {
201                                 L(String.format("ignoring duplicate account '%s'", accName));
202                                 continue;
203                             }
204                             String parentAccountName = LedgerAccount.extractParentName(accName);
205                             LedgerAccount parentAccount;
206                             if (parentAccountName != null) {
207                                 parentAccount = ensureAccountExists(parentAccountName, map,
208                                         syntheticAccounts);
209                             }
210                             else {
211                                 parentAccount = null;
212                             }
213                             lastAccount = new LedgerAccount(profile, accName, parentAccount);
214
215                             accounts.add(lastAccount);
216                             map.put(accName, lastAccount);
217
218                             state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
219                             L("→ expecting account amount");
220                         }
221                         break;
222
223                     case EXPECTING_ACCOUNT_AMOUNT:
224                         m = reAccountValue.matcher(line);
225                         boolean match_found = false;
226                         while (m.find()) {
227                             throwIfCancelled();
228
229                             match_found = true;
230                             String value = Objects.requireNonNull(m.group(1));
231                             String currency = m.group(2);
232                             if (currency == null)
233                                 currency = "";
234
235                             {
236                                 Matcher tmpM = reDecimalComma.matcher(value);
237                                 if (tmpM.find()) {
238                                     value = value.replace(".", "");
239                                     value = value.replace(',', '.');
240                                 }
241
242                                 tmpM = reDecimalPoint.matcher(value);
243                                 if (tmpM.find()) {
244                                     value = value.replace(",", "");
245                                     value = value.replace(" ", "");
246                                 }
247                             }
248                             L("curr=" + currency + ", value=" + value);
249                             final float val = Float.parseFloat(value);
250                             lastAccount.addAmount(val, currency);
251                             for (LedgerAccount syn : syntheticAccounts) {
252                                 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
253                                         currency, val, syn.getName()));
254                                 syn.addAmount(val, currency);
255                             }
256                         }
257
258                         if (match_found) {
259                             syntheticAccounts.clear();
260                             state = ParserState.EXPECTING_ACCOUNT;
261                             L("→ expecting account");
262                         }
263
264                         break;
265
266                     case EXPECTING_TRANSACTION:
267                         if (!line.isEmpty() && (line.charAt(0) == ' '))
268                             continue;
269                         m = reTransactionStart.matcher(line);
270                         if (m.find()) {
271                             transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
272                             state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
273                             L(String.format(Locale.ENGLISH,
274                                     "found transaction %d → expecting description", transactionId));
275                             progress.setProgress(++processedTransactionCount);
276                             if (maxTransactionId < transactionId)
277                                 maxTransactionId = transactionId;
278                             if ((progress.isIndeterminate()) ||
279                                 (progress.getTotal() < transactionId))
280                                 progress.setTotal(transactionId);
281                             publishProgress(progress);
282                         }
283                         m = reEnd.matcher(line);
284                         if (m.find()) {
285                             L("--- transaction value complete ---");
286                             break LINES;
287                         }
288                         break;
289
290                     case EXPECTING_TRANSACTION_DESCRIPTION:
291                         if (!line.isEmpty() && (line.charAt(0) == ' '))
292                             continue;
293                         m = reTransactionDescription.matcher(line);
294                         if (m.find()) {
295                             if (transactionId == 0)
296                                 throw new TransactionParserException(
297                                         "Transaction Id is 0 while expecting description");
298
299                             String date = Objects.requireNonNull(m.group(1));
300                             try {
301                                 int equalsIndex = date.indexOf('=');
302                                 if (equalsIndex >= 0)
303                                     date = date.substring(equalsIndex + 1);
304                                 transaction =
305                                         new LedgerTransaction(transactionId, date, m.group(2));
306                             }
307                             catch (ParseException e) {
308                                 throw new TransactionParserException(
309                                         String.format("Error parsing date '%s'", date));
310                             }
311                             state = ParserState.EXPECTING_TRANSACTION_DETAILS;
312                             L(String.format(Locale.ENGLISH,
313                                     "transaction %d created for %s (%s) →" + " expecting details",
314                                     transactionId, date, m.group(2)));
315                         }
316                         break;
317
318                     case EXPECTING_TRANSACTION_DETAILS:
319                         if (line.isEmpty()) {
320                             // transaction data collected
321
322                             transaction.finishLoading();
323                             transactions.add(transaction);
324
325                             state = ParserState.EXPECTING_TRANSACTION;
326                             L(String.format("transaction %s parsed → expecting transaction",
327                                     transaction.getId()));
328
329 // sounds like a good idea, but transaction-1 may not be the first one chronologically
330 // for example, when you add the initial seeding transaction after entering some others
331 //                                            if (transactionId == 1) {
332 //                                                L("This was the initial transaction.
333 //                                                Terminating " +
334 //                                                  "parser");
335 //                                                break LINES;
336 //                                            }
337                         }
338                         else {
339                             LedgerTransactionAccount lta = parseTransactionAccountLine(line);
340                             if (lta != null) {
341                                 transaction.addAccount(lta);
342                                 L(String.format(Locale.ENGLISH, "%d: %s = %s", transaction.getId(),
343                                         lta.getAccountName(), lta.getAmount()));
344                             }
345                             else
346                                 throw new IllegalStateException(
347                                         String.format("Can't parse transaction %d details: %s",
348                                                 transactionId, line));
349                         }
350                         break;
351                     default:
352                         throw new RuntimeException(
353                                 String.format("Unknown parser updating %s", state.name()));
354                 }
355             }
356
357             throwIfCancelled();
358         }
359     }
360     @NonNull
361     public LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
362                                              ArrayList<LedgerAccount> createdAccounts) {
363         LedgerAccount acc = map.get(accountName);
364
365         if (acc != null)
366             return acc;
367
368         String parentName = LedgerAccount.extractParentName(accountName);
369         LedgerAccount parentAccount;
370         if (parentName != null) {
371             parentAccount = ensureAccountExists(parentName, map, createdAccounts);
372         }
373         else {
374             parentAccount = null;
375         }
376
377         acc = new LedgerAccount(profile, accountName, parentAccount);
378         createdAccounts.add(acc);
379         return acc;
380     }
381     public void addNumberOfPostings(int number) {
382         expectedPostingsCount += number;
383     }
384     private List<LedgerAccount> retrieveAccountList() throws IOException, HTTPException {
385         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
386         http.setAllowUserInteraction(false);
387         switch (http.getResponseCode()) {
388             case 200:
389                 break;
390             case 404:
391                 return null;
392             default:
393                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
394         }
395         publishProgress(Progress.indeterminate());
396         SQLiteDatabase db = App.getDatabase();
397         ArrayList<LedgerAccount> list = new ArrayList<>();
398         HashMap<String, LedgerAccount> map = new HashMap<>();
399         HashMap<String, LedgerAccount> currentMap = new HashMap<>();
400         for (LedgerAccount acc : prevAccounts)
401             currentMap.put(acc.getName(), acc);
402         throwIfCancelled();
403         try (InputStream resp = http.getInputStream()) {
404             throwIfCancelled();
405             if (http.getResponseCode() != 200)
406                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
407
408             AccountListParser parser = new AccountListParser(resp);
409             expectedPostingsCount = 0;
410
411             while (true) {
412                 throwIfCancelled();
413                 LedgerAccount acc = parser.nextLedgerAccount(this, map);
414                 if (acc == null)
415                     break;
416                 list.add(acc);
417             }
418             throwIfCancelled();
419         }
420
421         // the current account tree may have changed, update the new-to be tree to match
422         for (LedgerAccount acc : list) {
423             LedgerAccount prevData = currentMap.get(acc.getName());
424             if (prevData != null) {
425                 acc.setExpanded(prevData.isExpanded());
426                 acc.setAmountsExpanded(prevData.amountsExpanded());
427             }
428         }
429
430         return list;
431     }
432     private List<LedgerTransaction> retrieveTransactionList()
433             throws IOException, ParseException, HTTPException {
434         Progress progress = new Progress();
435         progress.setTotal(expectedPostingsCount);
436
437         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
438         http.setAllowUserInteraction(false);
439         publishProgress(progress);
440         switch (http.getResponseCode()) {
441             case 200:
442                 break;
443             case 404:
444                 return null;
445             default:
446                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
447         }
448         ArrayList<LedgerTransaction> trList = new ArrayList<>();
449         try (InputStream resp = http.getInputStream()) {
450             throwIfCancelled();
451
452             TransactionListParser parser = new TransactionListParser(resp);
453
454             int processedPostings = 0;
455
456             while (true) {
457                 throwIfCancelled();
458                 ParsedLedgerTransaction parsedTransaction = parser.nextTransaction();
459                 throwIfCancelled();
460                 if (parsedTransaction == null)
461                     break;
462
463                 LedgerTransaction transaction = parsedTransaction.asLedgerTransaction();
464                 trList.add(transaction);
465
466                 progress.setProgress(processedPostings += transaction.getAccounts()
467                                                                      .size());
468 //                Logger.debug("trParser",
469 //                        String.format(Locale.US, "Parsed transaction %d - %s", transaction
470 //                        .getId(),
471 //                                transaction.getDescription()));
472 //                for (LedgerTransactionAccount acc : transaction.getAccounts()) {
473 //                    Logger.debug("trParser",
474 //                            String.format(Locale.US, "  %s", acc.getAccountName()));
475 //                }
476                 publishProgress(progress);
477             }
478
479             throwIfCancelled();
480         }
481
482         // json interface returns transactions if file order and the rest of the machinery
483         // expects them in reverse chronological order
484         Collections.sort(trList, (o1, o2) -> {
485             int res = o2.getDate()
486                         .compareTo(o1.getDate());
487             if (res != 0)
488                 return res;
489             return Integer.compare(o2.getId(), o1.getId());
490         });
491         return trList;
492     }
493
494     @SuppressLint("DefaultLocale")
495     @Override
496     protected Result doInBackground(Void... params) {
497         Data.backgroundTaskStarted();
498         List<LedgerAccount> accounts;
499         List<LedgerTransaction> transactions;
500         try {
501             accounts = retrieveAccountList();
502             if (accounts == null)
503                 transactions = null;
504             else
505                 transactions = retrieveTransactionList();
506             if (accounts == null || transactions == null) {
507                 accounts = new ArrayList<>();
508                 transactions = new ArrayList<>();
509                 retrieveTransactionListLegacy(accounts, transactions);
510             }
511             mainModel.setAndStoreAccountAndTransactionListFromWeb(accounts, transactions);
512
513             return new Result(accounts, transactions);
514         }
515         catch (MalformedURLException e) {
516             e.printStackTrace();
517             return new Result("Invalid server URL");
518         }
519         catch (HTTPException e) {
520             e.printStackTrace();
521             return new Result(String.format("HTTP error %d: %s", e.getResponseCode(),
522                     e.getResponseMessage()));
523         }
524         catch (IOException e) {
525             e.printStackTrace();
526             return new Result(e.getLocalizedMessage());
527         }
528         catch (ParseException e) {
529             e.printStackTrace();
530             return new Result("Network error");
531         }
532         catch (OperationCanceledException e) {
533             e.printStackTrace();
534             return new Result("Operation cancelled");
535         }
536         finally {
537             Data.backgroundTaskFinished();
538         }
539     }
540     public void throwIfCancelled() {
541         if (isCancelled())
542             throw new OperationCanceledException(null);
543     }
544     private enum ParserState {
545         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
546         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
547     }
548
549     public enum ProgressState {STARTING, RUNNING, FINISHED}
550
551     public static class Progress {
552         private int progress;
553         private int total;
554         private ProgressState state = ProgressState.RUNNING;
555         private String error = null;
556         private boolean indeterminate;
557         Progress() {
558             indeterminate = true;
559         }
560         Progress(int progress, int total) {
561             this.indeterminate = false;
562             this.progress = progress;
563             this.total = total;
564         }
565         public static Progress indeterminate() {
566             return new Progress();
567         }
568         public static Progress finished(String error) {
569             Progress p = new Progress();
570             p.setState(ProgressState.FINISHED);
571             p.setError(error);
572             return p;
573         }
574         public int getProgress() {
575             ensureState(ProgressState.RUNNING);
576             return progress;
577         }
578         protected void setProgress(int progress) {
579             this.progress = progress;
580             this.state = ProgressState.RUNNING;
581         }
582         public int getTotal() {
583             ensureState(ProgressState.RUNNING);
584             return total;
585         }
586         protected void setTotal(int total) {
587             this.total = total;
588             state = ProgressState.RUNNING;
589             indeterminate = total == -1;
590         }
591         private void ensureState(ProgressState wanted) {
592             if (state != wanted)
593                 throw new IllegalStateException(
594                         String.format("Bad state: %s, expected %s", state, wanted));
595         }
596         public ProgressState getState() {
597             return state;
598         }
599         public void setState(ProgressState state) {
600             this.state = state;
601         }
602         public String getError() {
603             ensureState(ProgressState.FINISHED);
604             return error;
605         }
606         public void setError(String error) {
607             this.error = error;
608             state = ProgressState.FINISHED;
609         }
610         public boolean isIndeterminate() {
611             return indeterminate;
612         }
613         public void setIndeterminate(boolean indeterminate) {
614             this.indeterminate = indeterminate;
615         }
616     }
617
618     private static class TransactionParserException extends IllegalStateException {
619         TransactionParserException(String message) {
620             super(message);
621         }
622     }
623
624     public static class Result {
625         public String error;
626         public List<LedgerAccount> accounts;
627         public List<LedgerTransaction> transactions;
628         Result(String error) {
629             this.error = error;
630         }
631         Result(List<LedgerAccount> accounts, List<LedgerTransaction> transactions) {
632             this.accounts = accounts;
633             this.transactions = transactions;
634         }
635     }
636 }