]> git.ktnx.net Git - mobile-ledger.git/blob - app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java
rework background progress to use MutableLiveData and observers
[mobile-ledger.git] / app / src / main / java / net / ktnx / mobileledger / async / RetrieveTransactionsTask.java
1 /*
2  * Copyright © 2020 Damyan Ivanov.
3  * This file is part of MoLe.
4  * MoLe is free software: you can distribute it and/or modify it
5  * under the term of the GNU General Public License as published by
6  * the Free Software Foundation, either version 3 of the License, or
7  * (at your opinion), any later version.
8  *
9  * MoLe is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  * GNU General Public License terms for details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
16  */
17
18 package net.ktnx.mobileledger.async;
19
20 import android.annotation.SuppressLint;
21 import android.database.sqlite.SQLiteDatabase;
22 import android.os.AsyncTask;
23 import android.os.OperationCanceledException;
24
25 import androidx.annotation.NonNull;
26
27 import net.ktnx.mobileledger.App;
28 import net.ktnx.mobileledger.err.HTTPException;
29 import net.ktnx.mobileledger.json.v1_15.AccountListParser;
30 import net.ktnx.mobileledger.json.v1_15.ParsedBalance;
31 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerAccount;
32 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerTransaction;
33 import net.ktnx.mobileledger.json.v1_15.TransactionListParser;
34 import net.ktnx.mobileledger.model.Data;
35 import net.ktnx.mobileledger.model.LedgerAccount;
36 import net.ktnx.mobileledger.model.LedgerTransaction;
37 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
38 import net.ktnx.mobileledger.model.MobileLedgerProfile;
39 import net.ktnx.mobileledger.ui.activity.MainActivity;
40 import net.ktnx.mobileledger.utils.NetworkUtil;
41
42 import java.io.BufferedReader;
43 import java.io.IOException;
44 import java.io.InputStream;
45 import java.io.InputStreamReader;
46 import java.lang.ref.WeakReference;
47 import java.net.HttpURLConnection;
48 import java.net.MalformedURLException;
49 import java.net.URLDecoder;
50 import java.nio.charset.StandardCharsets;
51 import java.text.ParseException;
52 import java.util.ArrayList;
53 import java.util.HashMap;
54 import java.util.Locale;
55 import java.util.Objects;
56 import java.util.regex.Matcher;
57 import java.util.regex.Pattern;
58
59
60 public class RetrieveTransactionsTask
61         extends AsyncTask<Void, RetrieveTransactionsTask.Progress, String> {
62     private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
63     private static final Pattern reComment = Pattern.compile("^\\s*;");
64     private static final Pattern reTransactionStart = Pattern.compile(
65             "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
66             "\"[^\"]*>([\\d.-]+)</td>");
67     private static final Pattern reTransactionDescription =
68             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
69     private static final Pattern reTransactionDetails = Pattern.compile(
70             "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
71             "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
72     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
73     private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
74     private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
75     private WeakReference<MainActivity> contextRef;
76     // %3A is '='
77     private Pattern reAccountName = Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
78     private Pattern reAccountValue = Pattern.compile(
79             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
80     private MobileLedgerProfile profile;
81     public RetrieveTransactionsTask(WeakReference<MainActivity> contextRef,
82                                     @NonNull MobileLedgerProfile profile) {
83         this.contextRef = contextRef;
84         this.profile = profile;
85     }
86     private static void L(String msg) {
87         //debug("transaction-parser", msg);
88     }
89     static LedgerTransactionAccount parseTransactionAccountLine(String line) {
90         Matcher m = reTransactionDetails.matcher(line);
91         if (m.find()) {
92             String postingStatus = m.group(1);
93             String acc_name = m.group(2);
94             String currencyPre = m.group(3);
95             String amount = Objects.requireNonNull(m.group(4));
96             String currencyPost = m.group(5);
97
98             String currency = null;
99             if ((currencyPre != null) && (currencyPre.length() > 0)) {
100                 if ((currencyPost != null) && (currencyPost.length() > 0))
101                     return null;
102                 currency = currencyPre;
103             }
104             else if ((currencyPost != null) && (currencyPost.length() > 0)) {
105                 currency = currencyPost;
106             }
107
108             amount = amount.replace(',', '.');
109
110             return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
111         }
112         else {
113             return null;
114         }
115     }
116     @Override
117     protected void onProgressUpdate(Progress... values) {
118         super.onProgressUpdate(values);
119         Data.backgroundTaskProgress.postValue(values[0]);
120     }
121     @Override
122     protected void onPreExecute() {
123         super.onPreExecute();
124         MainActivity context = getContext();
125         if (context == null)
126             return;
127         context.onRetrieveStart();
128     }
129     @Override
130     protected void onPostExecute(String error) {
131         super.onPostExecute(error);
132         Progress progress = new Progress();
133         progress.setState(ProgressState.FINISHED);
134         progress.setError(error);
135         onProgressUpdate(progress);
136     }
137     @Override
138     protected void onCancelled() {
139         super.onCancelled();
140         Progress progress = new Progress();
141         progress.setState(ProgressState.FINISHED);
142         onProgressUpdate(progress);
143     }
144     private String retrieveTransactionListLegacy() throws IOException, HTTPException {
145         Progress progress = Progress.indeterminate();
146         progress.setState(ProgressState.RUNNING);
147         int maxTransactionId = -1;
148         ArrayList<LedgerAccount> list = new ArrayList<>();
149         HashMap<String, LedgerAccount> map = new HashMap<>();
150         ArrayList<LedgerAccount> displayed = new ArrayList<>();
151         ArrayList<LedgerTransaction> transactions = new ArrayList<>();
152         LedgerAccount lastAccount = null;
153         ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
154
155         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
156         http.setAllowUserInteraction(false);
157         publishProgress(progress);
158         if (http.getResponseCode() != 200)
159             throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
160
161         try (InputStream resp = http.getInputStream()) {
162             if (http.getResponseCode() != 200)
163                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
164
165             int matchedTransactionsCount = 0;
166
167             ParserState state = ParserState.EXPECTING_ACCOUNT;
168             String line;
169             BufferedReader buf =
170                     new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
171
172             int processedTransactionCount = 0;
173             int transactionId = 0;
174             LedgerTransaction transaction = null;
175             LINES:
176             while ((line = buf.readLine()) != null) {
177                 throwIfCancelled();
178                 Matcher m;
179                 m = reComment.matcher(line);
180                 if (m.find()) {
181                     // TODO: comments are ignored for now
182 //                            Log.v("transaction-parser", "Ignoring comment");
183                     continue;
184                 }
185                 //L(String.format("State is %d", updating));
186                 switch (state) {
187                     case EXPECTING_ACCOUNT:
188                         if (line.equals("<h2>General Journal</h2>")) {
189                             state = ParserState.EXPECTING_TRANSACTION;
190                             L("→ expecting transaction");
191                             continue;
192                         }
193                         m = reAccountName.matcher(line);
194                         if (m.find()) {
195                             String acct_encoded = m.group(1);
196                             String accName = URLDecoder.decode(acct_encoded, "UTF-8");
197                             accName = accName.replace("\"", "");
198                             L(String.format("found account: %s", accName));
199
200                             lastAccount = map.get(accName);
201                             if (lastAccount != null) {
202                                 L(String.format("ignoring duplicate account '%s'", accName));
203                                 continue;
204                             }
205                             String parentAccountName = LedgerAccount.extractParentName(accName);
206                             LedgerAccount parentAccount;
207                             if (parentAccountName != null) {
208                                 parentAccount = ensureAccountExists(parentAccountName, map,
209                                         syntheticAccounts);
210                             }
211                             else {
212                                 parentAccount = null;
213                             }
214                             lastAccount = new LedgerAccount(profile, accName, parentAccount);
215
216                             list.add(lastAccount);
217                             map.put(accName, lastAccount);
218
219                             state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
220                             L("→ expecting account amount");
221                         }
222                         break;
223
224                     case EXPECTING_ACCOUNT_AMOUNT:
225                         m = reAccountValue.matcher(line);
226                         boolean match_found = false;
227                         while (m.find()) {
228                             throwIfCancelled();
229
230                             match_found = true;
231                             String value = Objects.requireNonNull(m.group(1));
232                             String currency = m.group(2);
233                             if (currency == null)
234                                 currency = "";
235
236                             {
237                                 Matcher tmpM = reDecimalComma.matcher(value);
238                                 if (tmpM.find()) {
239                                     value = value.replace(".", "");
240                                     value = value.replace(',', '.');
241                                 }
242
243                                 tmpM = reDecimalPoint.matcher(value);
244                                 if (tmpM.find()) {
245                                     value = value.replace(",", "");
246                                     value = value.replace(" ", "");
247                                 }
248                             }
249                             L("curr=" + currency + ", value=" + value);
250                             final float val = Float.parseFloat(value);
251                             lastAccount.addAmount(val, currency);
252                             for (LedgerAccount syn : syntheticAccounts) {
253                                 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
254                                         currency, val, syn.getName()));
255                                 syn.addAmount(val, currency);
256                             }
257                         }
258
259                         if (match_found) {
260                             syntheticAccounts.clear();
261                             state = ParserState.EXPECTING_ACCOUNT;
262                             L("→ expecting account");
263                         }
264
265                         break;
266
267                     case EXPECTING_TRANSACTION:
268                         if (!line.isEmpty() && (line.charAt(0) == ' '))
269                             continue;
270                         m = reTransactionStart.matcher(line);
271                         if (m.find()) {
272                             transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
273                             state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
274                             L(String.format(Locale.ENGLISH,
275                                     "found transaction %d → expecting description", transactionId));
276                             progress.setProgress(++processedTransactionCount);
277                             if (maxTransactionId < transactionId)
278                                 maxTransactionId = transactionId;
279                             if ((progress.isIndeterminate()) || (progress.getTotal() < transactionId))
280                                 progress.setTotal(transactionId);
281                             publishProgress(progress);
282                         }
283                         m = reEnd.matcher(line);
284                         if (m.find()) {
285                             L("--- transaction value complete ---");
286                             break LINES;
287                         }
288                         break;
289
290                     case EXPECTING_TRANSACTION_DESCRIPTION:
291                         if (!line.isEmpty() && (line.charAt(0) == ' '))
292                             continue;
293                         m = reTransactionDescription.matcher(line);
294                         if (m.find()) {
295                             if (transactionId == 0)
296                                 throw new TransactionParserException(
297                                         "Transaction Id is 0 while expecting " + "description");
298
299                             String date = Objects.requireNonNull(m.group(1));
300                             try {
301                                 int equalsIndex = date.indexOf('=');
302                                 if (equalsIndex >= 0)
303                                     date = date.substring(equalsIndex + 1);
304                                 transaction =
305                                         new LedgerTransaction(transactionId, date, m.group(2));
306                             }
307                             catch (ParseException e) {
308                                 e.printStackTrace();
309                                 return String.format("Error parsing date '%s'", date);
310                             }
311                             state = ParserState.EXPECTING_TRANSACTION_DETAILS;
312                             L(String.format(Locale.ENGLISH,
313                                     "transaction %d created for %s (%s) →" + " expecting details",
314                                     transactionId, date, m.group(2)));
315                         }
316                         break;
317
318                     case EXPECTING_TRANSACTION_DETAILS:
319                         if (line.isEmpty()) {
320                             // transaction data collected
321
322                             transaction.finishLoading();
323                             transactions.add(transaction);
324
325                             state = ParserState.EXPECTING_TRANSACTION;
326                             L(String.format("transaction %s parsed → expecting transaction",
327                                     transaction.getId()));
328
329 // sounds like a good idea, but transaction-1 may not be the first one chronologically
330 // for example, when you add the initial seeding transaction after entering some others
331 //                                            if (transactionId == 1) {
332 //                                                L("This was the initial transaction.
333 //                                                Terminating " +
334 //                                                  "parser");
335 //                                                break LINES;
336 //                                            }
337                         }
338                         else {
339                             LedgerTransactionAccount lta = parseTransactionAccountLine(line);
340                             if (lta != null) {
341                                 transaction.addAccount(lta);
342                                 L(String.format(Locale.ENGLISH, "%d: %s = %s", transaction.getId(),
343                                         lta.getAccountName(), lta.getAmount()));
344                             }
345                             else
346                                 throw new IllegalStateException(
347                                         String.format("Can't parse transaction %d details: %s",
348                                                 transactionId, line));
349                         }
350                         break;
351                     default:
352                         throw new RuntimeException(
353                                 String.format("Unknown parser updating %s", state.name()));
354                 }
355             }
356
357             throwIfCancelled();
358
359             profile.setAndStoreAccountAndTransactionListFromWeb(list, transactions);
360
361             return null;
362         }
363     }
364     private @NonNull
365     LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
366                                       ArrayList<LedgerAccount> createdAccounts) {
367         LedgerAccount acc = map.get(accountName);
368
369         if (acc != null)
370             return acc;
371
372         String parentName = LedgerAccount.extractParentName(accountName);
373         LedgerAccount parentAccount;
374         if (parentName != null) {
375             parentAccount = ensureAccountExists(parentName, map, createdAccounts);
376         }
377         else {
378             parentAccount = null;
379         }
380
381         acc = new LedgerAccount(profile, accountName, parentAccount);
382         createdAccounts.add(acc);
383         return acc;
384     }
385     private boolean retrieveAccountList() throws IOException, HTTPException {
386         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
387         http.setAllowUserInteraction(false);
388         switch (http.getResponseCode()) {
389             case 200:
390                 break;
391             case 404:
392                 return false;
393             default:
394                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
395         }
396         publishProgress(Progress.indeterminate());
397         SQLiteDatabase db = App.getDatabase();
398         ArrayList<LedgerAccount> list = new ArrayList<>();
399         HashMap<String, LedgerAccount> map = new HashMap<>();
400         HashMap<String, LedgerAccount> currentMap = new HashMap<>();
401         for (LedgerAccount acc : Objects.requireNonNull(profile.getAllAccounts()))
402             currentMap.put(acc.getName(), acc);
403         try (InputStream resp = http.getInputStream()) {
404             if (http.getResponseCode() != 200)
405                 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
406
407             AccountListParser parser = new AccountListParser(resp);
408
409             while (true) {
410                 throwIfCancelled();
411                 ParsedLedgerAccount parsedAccount = parser.nextAccount();
412                 if (parsedAccount == null) {
413                     break;
414                 }
415
416                 final String accName = parsedAccount.getAname();
417                 LedgerAccount acc = map.get(accName);
418                 if (acc != null)
419                     throw new RuntimeException(
420                             String.format("Account '%s' already present", acc.getName()));
421                 String parentName = LedgerAccount.extractParentName(accName);
422                 ArrayList<LedgerAccount> createdParents = new ArrayList<>();
423                 LedgerAccount parent;
424                 if (parentName == null) {
425                     parent = null;
426                 }
427                 else {
428                     parent = ensureAccountExists(parentName, map, createdParents);
429                     parent.setHasSubAccounts(true);
430                 }
431                 acc = new LedgerAccount(profile, accName, parent);
432                 list.add(acc);
433                 map.put(accName, acc);
434
435                 String lastCurrency = null;
436                 float lastCurrencyAmount = 0;
437                 for (ParsedBalance b : parsedAccount.getAibalance()) {
438                     throwIfCancelled();
439                     final String currency = b.getAcommodity();
440                     final float amount = b.getAquantity()
441                                           .asFloat();
442                     if (currency.equals(lastCurrency)) {
443                         lastCurrencyAmount += amount;
444                     }
445                     else {
446                         if (lastCurrency != null) {
447                             acc.addAmount(lastCurrencyAmount, lastCurrency);
448                         }
449                         lastCurrency = currency;
450                         lastCurrencyAmount = amount;
451                     }
452                 }
453                 if (lastCurrency != null) {
454                     acc.addAmount(lastCurrencyAmount, lastCurrency);
455                 }
456                 for (LedgerAccount p : createdParents)
457                     acc.propagateAmountsTo(p);
458             }
459             throwIfCancelled();
460         }
461
462         // the current account tree may have changed, update the new-to be tree to match
463         for (LedgerAccount acc : list) {
464             LedgerAccount prevData = currentMap.get(acc.getName());
465             if (prevData != null) {
466                 acc.setExpanded(prevData.isExpanded());
467                 acc.setAmountsExpanded(prevData.amountsExpanded());
468             }
469         }
470
471         profile.setAndStoreAccountListFromWeb(list);
472         return true;
473     }
474     private boolean retrieveTransactionList() throws IOException, ParseException, HTTPException {
475         Progress progress = new Progress();
476         int maxTransactionId = Data.transactions.size();
477
478         HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
479         http.setAllowUserInteraction(false);
480         publishProgress(progress);
481         switch (http.getResponseCode()) {
482             case 200:
483                 break;
484             case 404:
485                 return false;
486             default:
487                 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
488         }
489         try (InputStream resp = http.getInputStream()) {
490             throwIfCancelled();
491             ArrayList<LedgerTransaction> trList = new ArrayList<>();
492
493             TransactionListParser parser = new TransactionListParser(resp);
494
495             int processedTransactionCount = 0;
496
497             while (true) {
498                 throwIfCancelled();
499                 ParsedLedgerTransaction parsedTransaction = parser.nextTransaction();
500                 throwIfCancelled();
501                 if (parsedTransaction == null)
502                     break;
503
504                 LedgerTransaction transaction = parsedTransaction.asLedgerTransaction();
505                 trList.add(transaction);
506
507                 progress.setProgress(++processedTransactionCount);
508                 publishProgress(progress);
509             }
510
511             throwIfCancelled();
512             profile.setAndStoreTransactionList(trList);
513         }
514
515         return true;
516     }
517
518     @SuppressLint("DefaultLocale")
519     @Override
520     protected String doInBackground(Void... params) {
521         Data.backgroundTaskStarted();
522         try {
523             if (!retrieveAccountList() || !retrieveTransactionList())
524                 return retrieveTransactionListLegacy();
525             return null;
526         }
527         catch (MalformedURLException e) {
528             e.printStackTrace();
529             return "Invalid server URL";
530         }
531         catch (HTTPException e) {
532             e.printStackTrace();
533             return String.format("HTTP error %d: %s", e.getResponseCode(), e.getResponseMessage());
534         }
535         catch (IOException e) {
536             e.printStackTrace();
537             return e.getLocalizedMessage();
538         }
539         catch (ParseException e) {
540             e.printStackTrace();
541             return "Network error";
542         }
543         catch (OperationCanceledException e) {
544             e.printStackTrace();
545             return "Operation cancelled";
546         }
547         finally {
548             Data.backgroundTaskFinished();
549         }
550     }
551     private MainActivity getContext() {
552         return contextRef.get();
553     }
554     private void throwIfCancelled() {
555         if (isCancelled())
556             throw new OperationCanceledException(null);
557     }
558     private enum ParserState {
559         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
560         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
561     }
562
563     public enum ProgressState {STARTING, RUNNING, FINISHED}
564
565     public static class Progress {
566         private int progress;
567         private int total;
568         private ProgressState state = ProgressState.RUNNING;
569         private String error = null;
570         private boolean indeterminate;
571         Progress() {
572             indeterminate = true;
573         }
574         Progress(int progress, int total) {
575             this.indeterminate = false;
576             this.progress = progress;
577             this.total = total;
578         }
579         public static Progress indeterminate() {
580             return new Progress();
581         }
582         public static Progress finished(String error) {
583             Progress p = new Progress();
584             p.setState(ProgressState.FINISHED);
585             p.setError(error);
586             return p;
587         }
588         public int getProgress() {
589             ensureState(ProgressState.RUNNING);
590             return progress;
591         }
592         protected void setProgress(int progress) {
593             this.progress = progress;
594             this.state = ProgressState.RUNNING;
595         }
596         public int getTotal() {
597             ensureState(ProgressState.RUNNING);
598             return total;
599         }
600         protected void setTotal(int total) {
601             this.total = total;
602             state = ProgressState.RUNNING;
603         }
604         private void ensureState(ProgressState wanted) {
605             if (state != wanted)
606                 throw new IllegalStateException(
607                         String.format("Bad state: %s, expected %s", state, wanted));
608         }
609         public ProgressState getState() {
610             return state;
611         }
612         public void setState(ProgressState state) {
613             this.state = state;
614         }
615         public String getError() {
616             ensureState(ProgressState.FINISHED);
617             return error;
618         }
619         public void setError(String error) {
620             this.error = error;
621             state = ProgressState.FINISHED;
622         }
623         public boolean isIndeterminate() {
624             return indeterminate;
625         }
626         public void setIndeterminate(boolean indeterminate) {
627             this.indeterminate = indeterminate;
628         }
629     }
630
631     private static class TransactionParserException extends IllegalStateException {
632         TransactionParserException(String message) {
633             super(message);
634         }
635     }
636 }