2 * Copyright © 2020 Damyan Ivanov.
3 * This file is part of MoLe.
4 * MoLe is free software: you can distribute it and/or modify it
5 * under the term of the GNU General Public License as published by
6 * the Free Software Foundation, either version 3 of the License, or
7 * (at your opinion), any later version.
9 * MoLe is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License terms for details.
14 * You should have received a copy of the GNU General Public License
15 * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
18 package net.ktnx.mobileledger.async;
20 import android.annotation.SuppressLint;
21 import android.database.sqlite.SQLiteDatabase;
22 import android.os.AsyncTask;
23 import android.os.OperationCanceledException;
25 import androidx.annotation.NonNull;
27 import net.ktnx.mobileledger.App;
28 import net.ktnx.mobileledger.err.HTTPException;
29 import net.ktnx.mobileledger.json.v1_15.AccountListParser;
30 import net.ktnx.mobileledger.json.v1_15.ParsedBalance;
31 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerAccount;
32 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerTransaction;
33 import net.ktnx.mobileledger.json.v1_15.TransactionListParser;
34 import net.ktnx.mobileledger.model.Data;
35 import net.ktnx.mobileledger.model.LedgerAccount;
36 import net.ktnx.mobileledger.model.LedgerTransaction;
37 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
38 import net.ktnx.mobileledger.model.MobileLedgerProfile;
39 import net.ktnx.mobileledger.utils.NetworkUtil;
41 import java.io.BufferedReader;
42 import java.io.IOException;
43 import java.io.InputStream;
44 import java.io.InputStreamReader;
45 import java.net.HttpURLConnection;
46 import java.net.MalformedURLException;
47 import java.net.URLDecoder;
48 import java.nio.charset.StandardCharsets;
49 import java.text.ParseException;
50 import java.util.ArrayList;
51 import java.util.HashMap;
52 import java.util.Locale;
53 import java.util.Objects;
54 import java.util.regex.Matcher;
55 import java.util.regex.Pattern;
58 public class RetrieveTransactionsTask
59 extends AsyncTask<Void, RetrieveTransactionsTask.Progress, String> {
60 private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
61 private static final Pattern reComment = Pattern.compile("^\\s*;");
62 private static final Pattern reTransactionStart = Pattern.compile(
63 "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
64 "\"[^\"]*>([\\d.-]+)</td>");
65 private static final Pattern reTransactionDescription =
66 Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
67 private static final Pattern reTransactionDetails = Pattern.compile(
68 "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
69 "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
70 private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
71 private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
72 private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
74 private Pattern reAccountName = Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
75 private Pattern reAccountValue = Pattern.compile(
76 "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
77 private MobileLedgerProfile profile;
78 private int expectedPostingsCount = -1;
79 public RetrieveTransactionsTask(@NonNull MobileLedgerProfile profile) {
80 this.profile = profile;
82 private static void L(String msg) {
83 //debug("transaction-parser", msg);
85 static LedgerTransactionAccount parseTransactionAccountLine(String line) {
86 Matcher m = reTransactionDetails.matcher(line);
88 String postingStatus = m.group(1);
89 String acc_name = m.group(2);
90 String currencyPre = m.group(3);
91 String amount = Objects.requireNonNull(m.group(4));
92 String currencyPost = m.group(5);
94 String currency = null;
95 if ((currencyPre != null) && (currencyPre.length() > 0)) {
96 if ((currencyPost != null) && (currencyPost.length() > 0))
98 currency = currencyPre;
100 else if ((currencyPost != null) && (currencyPost.length() > 0)) {
101 currency = currencyPost;
104 amount = amount.replace(',', '.');
106 return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
113 protected void onProgressUpdate(Progress... values) {
114 super.onProgressUpdate(values);
115 Data.backgroundTaskProgress.postValue(values[0]);
118 protected void onPostExecute(String error) {
119 super.onPostExecute(error);
120 Progress progress = new Progress();
121 progress.setState(ProgressState.FINISHED);
122 progress.setError(error);
123 onProgressUpdate(progress);
126 protected void onCancelled() {
128 Progress progress = new Progress();
129 progress.setState(ProgressState.FINISHED);
130 onProgressUpdate(progress);
132 private String retrieveTransactionListLegacy() throws IOException, HTTPException {
133 Progress progress = Progress.indeterminate();
134 progress.setState(ProgressState.RUNNING);
135 progress.setTotal(expectedPostingsCount);
136 int maxTransactionId = -1;
137 ArrayList<LedgerAccount> list = new ArrayList<>();
138 HashMap<String, LedgerAccount> map = new HashMap<>();
139 ArrayList<LedgerAccount> displayed = new ArrayList<>();
140 ArrayList<LedgerTransaction> transactions = new ArrayList<>();
141 LedgerAccount lastAccount = null;
142 ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
144 HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
145 http.setAllowUserInteraction(false);
146 publishProgress(progress);
147 if (http.getResponseCode() != 200)
148 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
150 try (InputStream resp = http.getInputStream()) {
151 if (http.getResponseCode() != 200)
152 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
154 int matchedTransactionsCount = 0;
156 ParserState state = ParserState.EXPECTING_ACCOUNT;
159 new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
161 int processedTransactionCount = 0;
162 int transactionId = 0;
163 LedgerTransaction transaction = null;
165 while ((line = buf.readLine()) != null) {
168 m = reComment.matcher(line);
170 // TODO: comments are ignored for now
171 // Log.v("transaction-parser", "Ignoring comment");
174 //L(String.format("State is %d", updating));
176 case EXPECTING_ACCOUNT:
177 if (line.equals("<h2>General Journal</h2>")) {
178 state = ParserState.EXPECTING_TRANSACTION;
179 L("→ expecting transaction");
182 m = reAccountName.matcher(line);
184 String acct_encoded = m.group(1);
185 String accName = URLDecoder.decode(acct_encoded, "UTF-8");
186 accName = accName.replace("\"", "");
187 L(String.format("found account: %s", accName));
189 lastAccount = map.get(accName);
190 if (lastAccount != null) {
191 L(String.format("ignoring duplicate account '%s'", accName));
194 String parentAccountName = LedgerAccount.extractParentName(accName);
195 LedgerAccount parentAccount;
196 if (parentAccountName != null) {
197 parentAccount = ensureAccountExists(parentAccountName, map,
201 parentAccount = null;
203 lastAccount = new LedgerAccount(profile, accName, parentAccount);
205 list.add(lastAccount);
206 map.put(accName, lastAccount);
208 state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
209 L("→ expecting account amount");
213 case EXPECTING_ACCOUNT_AMOUNT:
214 m = reAccountValue.matcher(line);
215 boolean match_found = false;
220 String value = Objects.requireNonNull(m.group(1));
221 String currency = m.group(2);
222 if (currency == null)
226 Matcher tmpM = reDecimalComma.matcher(value);
228 value = value.replace(".", "");
229 value = value.replace(',', '.');
232 tmpM = reDecimalPoint.matcher(value);
234 value = value.replace(",", "");
235 value = value.replace(" ", "");
238 L("curr=" + currency + ", value=" + value);
239 final float val = Float.parseFloat(value);
240 lastAccount.addAmount(val, currency);
241 for (LedgerAccount syn : syntheticAccounts) {
242 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
243 currency, val, syn.getName()));
244 syn.addAmount(val, currency);
249 syntheticAccounts.clear();
250 state = ParserState.EXPECTING_ACCOUNT;
251 L("→ expecting account");
256 case EXPECTING_TRANSACTION:
257 if (!line.isEmpty() && (line.charAt(0) == ' '))
259 m = reTransactionStart.matcher(line);
261 transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
262 state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
263 L(String.format(Locale.ENGLISH,
264 "found transaction %d → expecting description", transactionId));
265 progress.setProgress(++processedTransactionCount);
266 if (maxTransactionId < transactionId)
267 maxTransactionId = transactionId;
268 if ((progress.isIndeterminate()) ||
269 (progress.getTotal() < transactionId))
270 progress.setTotal(transactionId);
271 publishProgress(progress);
273 m = reEnd.matcher(line);
275 L("--- transaction value complete ---");
280 case EXPECTING_TRANSACTION_DESCRIPTION:
281 if (!line.isEmpty() && (line.charAt(0) == ' '))
283 m = reTransactionDescription.matcher(line);
285 if (transactionId == 0)
286 throw new TransactionParserException(
287 "Transaction Id is 0 while expecting " + "description");
289 String date = Objects.requireNonNull(m.group(1));
291 int equalsIndex = date.indexOf('=');
292 if (equalsIndex >= 0)
293 date = date.substring(equalsIndex + 1);
295 new LedgerTransaction(transactionId, date, m.group(2));
297 catch (ParseException e) {
299 return String.format("Error parsing date '%s'", date);
301 state = ParserState.EXPECTING_TRANSACTION_DETAILS;
302 L(String.format(Locale.ENGLISH,
303 "transaction %d created for %s (%s) →" + " expecting details",
304 transactionId, date, m.group(2)));
308 case EXPECTING_TRANSACTION_DETAILS:
309 if (line.isEmpty()) {
310 // transaction data collected
312 transaction.finishLoading();
313 transactions.add(transaction);
315 state = ParserState.EXPECTING_TRANSACTION;
316 L(String.format("transaction %s parsed → expecting transaction",
317 transaction.getId()));
319 // sounds like a good idea, but transaction-1 may not be the first one chronologically
320 // for example, when you add the initial seeding transaction after entering some others
321 // if (transactionId == 1) {
322 // L("This was the initial transaction.
329 LedgerTransactionAccount lta = parseTransactionAccountLine(line);
331 transaction.addAccount(lta);
332 L(String.format(Locale.ENGLISH, "%d: %s = %s", transaction.getId(),
333 lta.getAccountName(), lta.getAmount()));
336 throw new IllegalStateException(
337 String.format("Can't parse transaction %d details: %s",
338 transactionId, line));
342 throw new RuntimeException(
343 String.format("Unknown parser updating %s", state.name()));
349 profile.setAndStoreAccountAndTransactionListFromWeb(list, transactions);
355 LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
356 ArrayList<LedgerAccount> createdAccounts) {
357 LedgerAccount acc = map.get(accountName);
362 String parentName = LedgerAccount.extractParentName(accountName);
363 LedgerAccount parentAccount;
364 if (parentName != null) {
365 parentAccount = ensureAccountExists(parentName, map, createdAccounts);
368 parentAccount = null;
371 acc = new LedgerAccount(profile, accountName, parentAccount);
372 createdAccounts.add(acc);
375 private boolean retrieveAccountList() throws IOException, HTTPException {
376 HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
377 http.setAllowUserInteraction(false);
378 switch (http.getResponseCode()) {
384 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
386 publishProgress(Progress.indeterminate());
387 SQLiteDatabase db = App.getDatabase();
388 ArrayList<LedgerAccount> list = new ArrayList<>();
389 HashMap<String, LedgerAccount> map = new HashMap<>();
390 HashMap<String, LedgerAccount> currentMap = new HashMap<>();
391 for (LedgerAccount acc : Objects.requireNonNull(profile.getAllAccounts()))
392 currentMap.put(acc.getName(), acc);
393 try (InputStream resp = http.getInputStream()) {
394 if (http.getResponseCode() != 200)
395 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
397 AccountListParser parser = new AccountListParser(resp);
398 expectedPostingsCount = 0;
402 ParsedLedgerAccount parsedAccount = parser.nextAccount();
403 if (parsedAccount == null) {
406 expectedPostingsCount += parsedAccount.getAnumpostings();
407 final String accName = parsedAccount.getAname();
408 LedgerAccount acc = map.get(accName);
410 throw new RuntimeException(
411 String.format("Account '%s' already present", acc.getName()));
412 String parentName = LedgerAccount.extractParentName(accName);
413 ArrayList<LedgerAccount> createdParents = new ArrayList<>();
414 LedgerAccount parent;
415 if (parentName == null) {
419 parent = ensureAccountExists(parentName, map, createdParents);
420 parent.setHasSubAccounts(true);
422 acc = new LedgerAccount(profile, accName, parent);
424 map.put(accName, acc);
426 String lastCurrency = null;
427 float lastCurrencyAmount = 0;
428 for (ParsedBalance b : parsedAccount.getAibalance()) {
430 final String currency = b.getAcommodity();
431 final float amount = b.getAquantity()
433 if (currency.equals(lastCurrency)) {
434 lastCurrencyAmount += amount;
437 if (lastCurrency != null) {
438 acc.addAmount(lastCurrencyAmount, lastCurrency);
440 lastCurrency = currency;
441 lastCurrencyAmount = amount;
444 if (lastCurrency != null) {
445 acc.addAmount(lastCurrencyAmount, lastCurrency);
447 for (LedgerAccount p : createdParents)
448 acc.propagateAmountsTo(p);
453 // the current account tree may have changed, update the new-to be tree to match
454 for (LedgerAccount acc : list) {
455 LedgerAccount prevData = currentMap.get(acc.getName());
456 if (prevData != null) {
457 acc.setExpanded(prevData.isExpanded());
458 acc.setAmountsExpanded(prevData.amountsExpanded());
462 profile.setAndStoreAccountListFromWeb(list);
465 private boolean retrieveTransactionList() throws IOException, ParseException, HTTPException {
466 Progress progress = new Progress();
467 int maxTransactionId = Data.transactions.size();
468 progress.setTotal(expectedPostingsCount);
470 HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
471 http.setAllowUserInteraction(false);
472 publishProgress(progress);
473 switch (http.getResponseCode()) {
479 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
481 try (InputStream resp = http.getInputStream()) {
483 ArrayList<LedgerTransaction> trList = new ArrayList<>();
485 TransactionListParser parser = new TransactionListParser(resp);
487 int processedPostings = 0;
491 ParsedLedgerTransaction parsedTransaction = parser.nextTransaction();
493 if (parsedTransaction == null)
496 LedgerTransaction transaction = parsedTransaction.asLedgerTransaction();
497 trList.add(transaction);
499 progress.setProgress(processedPostings += transaction.getAccounts()
501 publishProgress(progress);
505 profile.setAndStoreTransactionList(trList);
511 @SuppressLint("DefaultLocale")
513 protected String doInBackground(Void... params) {
514 Data.backgroundTaskStarted();
516 if (!retrieveAccountList() || !retrieveTransactionList())
517 return retrieveTransactionListLegacy();
520 catch (MalformedURLException e) {
522 return "Invalid server URL";
524 catch (HTTPException e) {
526 return String.format("HTTP error %d: %s", e.getResponseCode(), e.getResponseMessage());
528 catch (IOException e) {
530 return e.getLocalizedMessage();
532 catch (ParseException e) {
534 return "Network error";
536 catch (OperationCanceledException e) {
538 return "Operation cancelled";
541 Data.backgroundTaskFinished();
544 private void throwIfCancelled() {
546 throw new OperationCanceledException(null);
548 private enum ParserState {
549 EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
550 EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
553 public enum ProgressState {STARTING, RUNNING, FINISHED}
555 public static class Progress {
556 private int progress;
558 private ProgressState state = ProgressState.RUNNING;
559 private String error = null;
560 private boolean indeterminate;
562 indeterminate = true;
564 Progress(int progress, int total) {
565 this.indeterminate = false;
566 this.progress = progress;
569 public static Progress indeterminate() {
570 return new Progress();
572 public static Progress finished(String error) {
573 Progress p = new Progress();
574 p.setState(ProgressState.FINISHED);
578 public int getProgress() {
579 ensureState(ProgressState.RUNNING);
582 protected void setProgress(int progress) {
583 this.progress = progress;
584 this.state = ProgressState.RUNNING;
586 public int getTotal() {
587 ensureState(ProgressState.RUNNING);
590 protected void setTotal(int total) {
592 state = ProgressState.RUNNING;
593 indeterminate = total == -1;
595 private void ensureState(ProgressState wanted) {
597 throw new IllegalStateException(
598 String.format("Bad state: %s, expected %s", state, wanted));
600 public ProgressState getState() {
603 public void setState(ProgressState state) {
606 public String getError() {
607 ensureState(ProgressState.FINISHED);
610 public void setError(String error) {
612 state = ProgressState.FINISHED;
614 public boolean isIndeterminate() {
615 return indeterminate;
617 public void setIndeterminate(boolean indeterminate) {
618 this.indeterminate = indeterminate;
622 private static class TransactionParserException extends IllegalStateException {
623 TransactionParserException(String message) {