2 * Copyright © 2020 Damyan Ivanov.
3 * This file is part of MoLe.
4 * MoLe is free software: you can distribute it and/or modify it
5 * under the term of the GNU General Public License as published by
6 * the Free Software Foundation, either version 3 of the License, or
7 * (at your opinion), any later version.
9 * MoLe is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License terms for details.
14 * You should have received a copy of the GNU General Public License
15 * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
18 package net.ktnx.mobileledger.async;
20 import android.annotation.SuppressLint;
21 import android.database.sqlite.SQLiteDatabase;
22 import android.os.AsyncTask;
23 import android.os.OperationCanceledException;
25 import androidx.annotation.NonNull;
27 import net.ktnx.mobileledger.App;
28 import net.ktnx.mobileledger.err.HTTPException;
29 import net.ktnx.mobileledger.json.v1_15.AccountListParser;
30 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerTransaction;
31 import net.ktnx.mobileledger.json.v1_15.TransactionListParser;
32 import net.ktnx.mobileledger.model.Data;
33 import net.ktnx.mobileledger.model.LedgerAccount;
34 import net.ktnx.mobileledger.model.LedgerTransaction;
35 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
36 import net.ktnx.mobileledger.model.MobileLedgerProfile;
37 import net.ktnx.mobileledger.ui.MainModel;
38 import net.ktnx.mobileledger.utils.NetworkUtil;
40 import java.io.BufferedReader;
41 import java.io.IOException;
42 import java.io.InputStream;
43 import java.io.InputStreamReader;
44 import java.net.HttpURLConnection;
45 import java.net.MalformedURLException;
46 import java.net.URLDecoder;
47 import java.nio.charset.StandardCharsets;
48 import java.text.ParseException;
49 import java.util.ArrayList;
50 import java.util.Collections;
51 import java.util.HashMap;
52 import java.util.List;
53 import java.util.Locale;
54 import java.util.Objects;
55 import java.util.regex.Matcher;
56 import java.util.regex.Pattern;
59 public class RetrieveTransactionsTask extends
60 AsyncTask<Void, RetrieveTransactionsTask.Progress, RetrieveTransactionsTask.Result> {
61 private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
62 private static final Pattern reComment = Pattern.compile("^\\s*;");
63 private static final Pattern reTransactionStart = Pattern.compile(
64 "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
65 "\"[^\"]*>([\\d.-]+)</td>");
66 private static final Pattern reTransactionDescription =
67 Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
68 private static final Pattern reTransactionDetails = Pattern.compile(
69 "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
70 "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
71 private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
72 private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
73 private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
75 private final Pattern reAccountName =
76 Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
77 private final Pattern reAccountValue = Pattern.compile(
78 "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
79 private final MainModel mainModel;
80 private final MobileLedgerProfile profile;
81 private final List<LedgerAccount> prevAccounts;
82 private int expectedPostingsCount = -1;
83 public RetrieveTransactionsTask(@NonNull MainModel mainModel,
84 @NonNull MobileLedgerProfile profile,
85 List<LedgerAccount> accounts) {
86 this.mainModel = mainModel;
87 this.profile = profile;
88 this.prevAccounts = accounts;
90 private static void L(String msg) {
91 //debug("transaction-parser", msg);
93 static LedgerTransactionAccount parseTransactionAccountLine(String line) {
94 Matcher m = reTransactionDetails.matcher(line);
96 String postingStatus = m.group(1);
97 String acc_name = m.group(2);
98 String currencyPre = m.group(3);
99 String amount = Objects.requireNonNull(m.group(4));
100 String currencyPost = m.group(5);
102 String currency = null;
103 if ((currencyPre != null) && (currencyPre.length() > 0)) {
104 if ((currencyPost != null) && (currencyPost.length() > 0))
106 currency = currencyPre;
108 else if ((currencyPost != null) && (currencyPost.length() > 0)) {
109 currency = currencyPost;
112 amount = amount.replace(',', '.');
114 return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
120 public MobileLedgerProfile getProfile() {
124 protected void onProgressUpdate(Progress... values) {
125 super.onProgressUpdate(values);
126 Data.backgroundTaskProgress.postValue(values[0]);
129 protected void onPostExecute(Result result) {
130 super.onPostExecute(result);
131 Progress progress = new Progress();
132 progress.setState(ProgressState.FINISHED);
133 progress.setError(result.error);
134 onProgressUpdate(progress);
137 protected void onCancelled() {
139 Progress progress = new Progress();
140 progress.setState(ProgressState.FINISHED);
141 onProgressUpdate(progress);
143 private void retrieveTransactionListLegacy(List<LedgerAccount> accounts,
144 List<LedgerTransaction> transactions)
145 throws IOException, HTTPException {
146 Progress progress = Progress.indeterminate();
147 progress.setState(ProgressState.RUNNING);
148 progress.setTotal(expectedPostingsCount);
149 int maxTransactionId = -1;
150 HashMap<String, LedgerAccount> map = new HashMap<>();
151 LedgerAccount lastAccount = null;
152 ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
154 HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
155 http.setAllowUserInteraction(false);
156 publishProgress(progress);
157 if (http.getResponseCode() != 200)
158 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
160 try (InputStream resp = http.getInputStream()) {
161 if (http.getResponseCode() != 200)
162 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
164 int matchedTransactionsCount = 0;
166 ParserState state = ParserState.EXPECTING_ACCOUNT;
169 new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
171 int processedTransactionCount = 0;
172 int transactionId = 0;
173 LedgerTransaction transaction = null;
175 while ((line = buf.readLine()) != null) {
178 m = reComment.matcher(line);
180 // TODO: comments are ignored for now
181 // Log.v("transaction-parser", "Ignoring comment");
184 //L(String.format("State is %d", updating));
186 case EXPECTING_ACCOUNT:
187 if (line.equals("<h2>General Journal</h2>")) {
188 state = ParserState.EXPECTING_TRANSACTION;
189 L("→ expecting transaction");
192 m = reAccountName.matcher(line);
194 String acct_encoded = m.group(1);
195 String accName = URLDecoder.decode(acct_encoded, "UTF-8");
196 accName = accName.replace("\"", "");
197 L(String.format("found account: %s", accName));
199 lastAccount = map.get(accName);
200 if (lastAccount != null) {
201 L(String.format("ignoring duplicate account '%s'", accName));
204 String parentAccountName = LedgerAccount.extractParentName(accName);
205 LedgerAccount parentAccount;
206 if (parentAccountName != null) {
207 parentAccount = ensureAccountExists(parentAccountName, map,
211 parentAccount = null;
213 lastAccount = new LedgerAccount(profile, accName, parentAccount);
215 accounts.add(lastAccount);
216 map.put(accName, lastAccount);
218 state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
219 L("→ expecting account amount");
223 case EXPECTING_ACCOUNT_AMOUNT:
224 m = reAccountValue.matcher(line);
225 boolean match_found = false;
230 String value = Objects.requireNonNull(m.group(1));
231 String currency = m.group(2);
232 if (currency == null)
236 Matcher tmpM = reDecimalComma.matcher(value);
238 value = value.replace(".", "");
239 value = value.replace(',', '.');
242 tmpM = reDecimalPoint.matcher(value);
244 value = value.replace(",", "");
245 value = value.replace(" ", "");
248 L("curr=" + currency + ", value=" + value);
249 final float val = Float.parseFloat(value);
250 lastAccount.addAmount(val, currency);
251 for (LedgerAccount syn : syntheticAccounts) {
252 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
253 currency, val, syn.getName()));
254 syn.addAmount(val, currency);
259 syntheticAccounts.clear();
260 state = ParserState.EXPECTING_ACCOUNT;
261 L("→ expecting account");
266 case EXPECTING_TRANSACTION:
267 if (!line.isEmpty() && (line.charAt(0) == ' '))
269 m = reTransactionStart.matcher(line);
271 transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
272 state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
273 L(String.format(Locale.ENGLISH,
274 "found transaction %d → expecting description", transactionId));
275 progress.setProgress(++processedTransactionCount);
276 if (maxTransactionId < transactionId)
277 maxTransactionId = transactionId;
278 if ((progress.isIndeterminate()) ||
279 (progress.getTotal() < transactionId))
280 progress.setTotal(transactionId);
281 publishProgress(progress);
283 m = reEnd.matcher(line);
285 L("--- transaction value complete ---");
290 case EXPECTING_TRANSACTION_DESCRIPTION:
291 if (!line.isEmpty() && (line.charAt(0) == ' '))
293 m = reTransactionDescription.matcher(line);
295 if (transactionId == 0)
296 throw new TransactionParserException(
297 "Transaction Id is 0 while expecting description");
299 String date = Objects.requireNonNull(m.group(1));
301 int equalsIndex = date.indexOf('=');
302 if (equalsIndex >= 0)
303 date = date.substring(equalsIndex + 1);
305 new LedgerTransaction(transactionId, date, m.group(2));
307 catch (ParseException e) {
308 throw new TransactionParserException(
309 String.format("Error parsing date '%s'", date));
311 state = ParserState.EXPECTING_TRANSACTION_DETAILS;
312 L(String.format(Locale.ENGLISH,
313 "transaction %d created for %s (%s) →" + " expecting details",
314 transactionId, date, m.group(2)));
318 case EXPECTING_TRANSACTION_DETAILS:
319 if (line.isEmpty()) {
320 // transaction data collected
322 transaction.finishLoading();
323 transactions.add(transaction);
325 state = ParserState.EXPECTING_TRANSACTION;
326 L(String.format("transaction %s parsed → expecting transaction",
327 transaction.getId()));
329 // sounds like a good idea, but transaction-1 may not be the first one chronologically
330 // for example, when you add the initial seeding transaction after entering some others
331 // if (transactionId == 1) {
332 // L("This was the initial transaction.
339 LedgerTransactionAccount lta = parseTransactionAccountLine(line);
341 transaction.addAccount(lta);
342 L(String.format(Locale.ENGLISH, "%d: %s = %s", transaction.getId(),
343 lta.getAccountName(), lta.getAmount()));
346 throw new IllegalStateException(
347 String.format("Can't parse transaction %d details: %s",
348 transactionId, line));
352 throw new RuntimeException(
353 String.format("Unknown parser updating %s", state.name()));
361 public LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
362 ArrayList<LedgerAccount> createdAccounts) {
363 LedgerAccount acc = map.get(accountName);
368 String parentName = LedgerAccount.extractParentName(accountName);
369 LedgerAccount parentAccount;
370 if (parentName != null) {
371 parentAccount = ensureAccountExists(parentName, map, createdAccounts);
374 parentAccount = null;
377 acc = new LedgerAccount(profile, accountName, parentAccount);
378 createdAccounts.add(acc);
381 public void addNumberOfPostings(int number) {
382 expectedPostingsCount += number;
384 private List<LedgerAccount> retrieveAccountList() throws IOException, HTTPException {
385 HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
386 http.setAllowUserInteraction(false);
387 switch (http.getResponseCode()) {
393 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
395 publishProgress(Progress.indeterminate());
396 SQLiteDatabase db = App.getDatabase();
397 ArrayList<LedgerAccount> list = new ArrayList<>();
398 HashMap<String, LedgerAccount> map = new HashMap<>();
399 HashMap<String, LedgerAccount> currentMap = new HashMap<>();
400 for (LedgerAccount acc : prevAccounts)
401 currentMap.put(acc.getName(), acc);
403 try (InputStream resp = http.getInputStream()) {
405 if (http.getResponseCode() != 200)
406 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
408 AccountListParser parser = new AccountListParser(resp);
409 expectedPostingsCount = 0;
413 LedgerAccount acc = parser.nextLedgerAccount(this, map);
421 // the current account tree may have changed, update the new-to be tree to match
422 for (LedgerAccount acc : list) {
423 LedgerAccount prevData = currentMap.get(acc.getName());
424 if (prevData != null) {
425 acc.setExpanded(prevData.isExpanded());
426 acc.setAmountsExpanded(prevData.amountsExpanded());
432 private List<LedgerTransaction> retrieveTransactionList()
433 throws IOException, ParseException, HTTPException {
434 Progress progress = new Progress();
435 progress.setTotal(expectedPostingsCount);
437 HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
438 http.setAllowUserInteraction(false);
439 publishProgress(progress);
440 switch (http.getResponseCode()) {
446 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
448 ArrayList<LedgerTransaction> trList = new ArrayList<>();
449 try (InputStream resp = http.getInputStream()) {
452 TransactionListParser parser = new TransactionListParser(resp);
454 int processedPostings = 0;
458 ParsedLedgerTransaction parsedTransaction = parser.nextTransaction();
460 if (parsedTransaction == null)
463 LedgerTransaction transaction = parsedTransaction.asLedgerTransaction();
464 trList.add(transaction);
466 progress.setProgress(processedPostings += transaction.getAccounts()
468 // Logger.debug("trParser",
469 // String.format(Locale.US, "Parsed transaction %d - %s", transaction
471 // transaction.getDescription()));
472 // for (LedgerTransactionAccount acc : transaction.getAccounts()) {
473 // Logger.debug("trParser",
474 // String.format(Locale.US, " %s", acc.getAccountName()));
476 publishProgress(progress);
482 // json interface returns transactions if file order and the rest of the machinery
483 // expects them in reverse chronological order
484 Collections.sort(trList, (o1, o2) -> {
485 int res = o2.getDate()
486 .compareTo(o1.getDate());
489 return Integer.compare(o2.getId(), o1.getId());
494 @SuppressLint("DefaultLocale")
496 protected Result doInBackground(Void... params) {
497 Data.backgroundTaskStarted();
498 List<LedgerAccount> accounts;
499 List<LedgerTransaction> transactions;
501 accounts = retrieveAccountList();
502 if (accounts == null)
505 transactions = retrieveTransactionList();
506 if (accounts == null || transactions == null) {
507 accounts = new ArrayList<>();
508 transactions = new ArrayList<>();
509 retrieveTransactionListLegacy(accounts, transactions);
511 mainModel.setAndStoreAccountAndTransactionListFromWeb(accounts, transactions);
513 return new Result(accounts, transactions);
515 catch (MalformedURLException e) {
517 return new Result("Invalid server URL");
519 catch (HTTPException e) {
521 return new Result(String.format("HTTP error %d: %s", e.getResponseCode(),
522 e.getResponseMessage()));
524 catch (IOException e) {
526 return new Result(e.getLocalizedMessage());
528 catch (ParseException e) {
530 return new Result("Network error");
532 catch (OperationCanceledException e) {
534 return new Result("Operation cancelled");
537 Data.backgroundTaskFinished();
540 public void throwIfCancelled() {
542 throw new OperationCanceledException(null);
544 private enum ParserState {
545 EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
546 EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
549 public enum ProgressState {STARTING, RUNNING, FINISHED}
551 public static class Progress {
552 private int progress;
554 private ProgressState state = ProgressState.RUNNING;
555 private String error = null;
556 private boolean indeterminate;
558 indeterminate = true;
560 Progress(int progress, int total) {
561 this.indeterminate = false;
562 this.progress = progress;
565 public static Progress indeterminate() {
566 return new Progress();
568 public static Progress finished(String error) {
569 Progress p = new Progress();
570 p.setState(ProgressState.FINISHED);
574 public int getProgress() {
575 ensureState(ProgressState.RUNNING);
578 protected void setProgress(int progress) {
579 this.progress = progress;
580 this.state = ProgressState.RUNNING;
582 public int getTotal() {
583 ensureState(ProgressState.RUNNING);
586 protected void setTotal(int total) {
588 state = ProgressState.RUNNING;
589 indeterminate = total == -1;
591 private void ensureState(ProgressState wanted) {
593 throw new IllegalStateException(
594 String.format("Bad state: %s, expected %s", state, wanted));
596 public ProgressState getState() {
599 public void setState(ProgressState state) {
602 public String getError() {
603 ensureState(ProgressState.FINISHED);
606 public void setError(String error) {
608 state = ProgressState.FINISHED;
610 public boolean isIndeterminate() {
611 return indeterminate;
613 public void setIndeterminate(boolean indeterminate) {
614 this.indeterminate = indeterminate;
618 private static class TransactionParserException extends IllegalStateException {
619 TransactionParserException(String message) {
624 public static class Result {
626 public List<LedgerAccount> accounts;
627 public List<LedgerTransaction> transactions;
628 Result(String error) {
631 Result(List<LedgerAccount> accounts, List<LedgerTransaction> transactions) {
632 this.accounts = accounts;
633 this.transactions = transactions;