2 * Copyright © 2020 Damyan Ivanov.
3 * This file is part of MoLe.
4 * MoLe is free software: you can distribute it and/or modify it
5 * under the term of the GNU General Public License as published by
6 * the Free Software Foundation, either version 3 of the License, or
7 * (at your opinion), any later version.
9 * MoLe is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License terms for details.
14 * You should have received a copy of the GNU General Public License
15 * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
18 package net.ktnx.mobileledger.async;
20 import android.annotation.SuppressLint;
21 import android.database.sqlite.SQLiteDatabase;
22 import android.os.AsyncTask;
23 import android.os.OperationCanceledException;
25 import androidx.annotation.NonNull;
27 import net.ktnx.mobileledger.App;
28 import net.ktnx.mobileledger.err.HTTPException;
29 import net.ktnx.mobileledger.json.v1_15.AccountListParser;
30 import net.ktnx.mobileledger.json.v1_15.ParsedBalance;
31 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerAccount;
32 import net.ktnx.mobileledger.json.v1_15.ParsedLedgerTransaction;
33 import net.ktnx.mobileledger.json.v1_15.TransactionListParser;
34 import net.ktnx.mobileledger.model.Data;
35 import net.ktnx.mobileledger.model.LedgerAccount;
36 import net.ktnx.mobileledger.model.LedgerTransaction;
37 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
38 import net.ktnx.mobileledger.model.MobileLedgerProfile;
39 import net.ktnx.mobileledger.utils.NetworkUtil;
41 import java.io.BufferedReader;
42 import java.io.IOException;
43 import java.io.InputStream;
44 import java.io.InputStreamReader;
45 import java.net.HttpURLConnection;
46 import java.net.MalformedURLException;
47 import java.net.URLDecoder;
48 import java.nio.charset.StandardCharsets;
49 import java.text.ParseException;
50 import java.util.ArrayList;
51 import java.util.HashMap;
52 import java.util.Locale;
53 import java.util.Objects;
54 import java.util.regex.Matcher;
55 import java.util.regex.Pattern;
58 public class RetrieveTransactionsTask
59 extends AsyncTask<Void, RetrieveTransactionsTask.Progress, String> {
60 private static final int MATCHING_TRANSACTIONS_LIMIT = 150;
61 private static final Pattern reComment = Pattern.compile("^\\s*;");
62 private static final Pattern reTransactionStart = Pattern.compile(
63 "<tr class=\"title\" " + "id=\"transaction-(\\d+)" + "\"><td class=\"date" +
64 "\"[^\"]*>([\\d.-]+)</td>");
65 private static final Pattern reTransactionDescription =
66 Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
67 private static final Pattern reTransactionDetails = Pattern.compile(
68 "^\\s+" + "([!*]\\s+)?" + "(\\S[\\S\\s]+\\S)\\s\\s+" + "(?:([^\\d\\s+\\-]+)\\s*)?" +
69 "([-+]?\\d[\\d,.]*)" + "(?:\\s*([^\\d\\s+\\-]+)\\s*$)?");
70 private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
71 private static final Pattern reDecimalPoint = Pattern.compile("\\.\\d\\d?$");
72 private static final Pattern reDecimalComma = Pattern.compile(",\\d\\d?$");
74 private Pattern reAccountName = Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
75 private Pattern reAccountValue = Pattern.compile(
76 "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
77 private MobileLedgerProfile profile;
78 public RetrieveTransactionsTask(@NonNull MobileLedgerProfile profile) {
79 this.profile = profile;
81 private static void L(String msg) {
82 //debug("transaction-parser", msg);
84 static LedgerTransactionAccount parseTransactionAccountLine(String line) {
85 Matcher m = reTransactionDetails.matcher(line);
87 String postingStatus = m.group(1);
88 String acc_name = m.group(2);
89 String currencyPre = m.group(3);
90 String amount = Objects.requireNonNull(m.group(4));
91 String currencyPost = m.group(5);
93 String currency = null;
94 if ((currencyPre != null) && (currencyPre.length() > 0)) {
95 if ((currencyPost != null) && (currencyPost.length() > 0))
97 currency = currencyPre;
99 else if ((currencyPost != null) && (currencyPost.length() > 0)) {
100 currency = currencyPost;
103 amount = amount.replace(',', '.');
105 return new LedgerTransactionAccount(acc_name, Float.parseFloat(amount), currency, null);
112 protected void onProgressUpdate(Progress... values) {
113 super.onProgressUpdate(values);
114 Data.backgroundTaskProgress.postValue(values[0]);
117 protected void onPostExecute(String error) {
118 super.onPostExecute(error);
119 Progress progress = new Progress();
120 progress.setState(ProgressState.FINISHED);
121 progress.setError(error);
122 onProgressUpdate(progress);
125 protected void onCancelled() {
127 Progress progress = new Progress();
128 progress.setState(ProgressState.FINISHED);
129 onProgressUpdate(progress);
131 private String retrieveTransactionListLegacy() throws IOException, HTTPException {
132 Progress progress = Progress.indeterminate();
133 progress.setState(ProgressState.RUNNING);
134 int maxTransactionId = -1;
135 ArrayList<LedgerAccount> list = new ArrayList<>();
136 HashMap<String, LedgerAccount> map = new HashMap<>();
137 ArrayList<LedgerAccount> displayed = new ArrayList<>();
138 ArrayList<LedgerTransaction> transactions = new ArrayList<>();
139 LedgerAccount lastAccount = null;
140 ArrayList<LedgerAccount> syntheticAccounts = new ArrayList<>();
142 HttpURLConnection http = NetworkUtil.prepareConnection(profile, "journal");
143 http.setAllowUserInteraction(false);
144 publishProgress(progress);
145 if (http.getResponseCode() != 200)
146 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
148 try (InputStream resp = http.getInputStream()) {
149 if (http.getResponseCode() != 200)
150 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
152 int matchedTransactionsCount = 0;
154 ParserState state = ParserState.EXPECTING_ACCOUNT;
157 new BufferedReader(new InputStreamReader(resp, StandardCharsets.UTF_8));
159 int processedTransactionCount = 0;
160 int transactionId = 0;
161 LedgerTransaction transaction = null;
163 while ((line = buf.readLine()) != null) {
166 m = reComment.matcher(line);
168 // TODO: comments are ignored for now
169 // Log.v("transaction-parser", "Ignoring comment");
172 //L(String.format("State is %d", updating));
174 case EXPECTING_ACCOUNT:
175 if (line.equals("<h2>General Journal</h2>")) {
176 state = ParserState.EXPECTING_TRANSACTION;
177 L("→ expecting transaction");
180 m = reAccountName.matcher(line);
182 String acct_encoded = m.group(1);
183 String accName = URLDecoder.decode(acct_encoded, "UTF-8");
184 accName = accName.replace("\"", "");
185 L(String.format("found account: %s", accName));
187 lastAccount = map.get(accName);
188 if (lastAccount != null) {
189 L(String.format("ignoring duplicate account '%s'", accName));
192 String parentAccountName = LedgerAccount.extractParentName(accName);
193 LedgerAccount parentAccount;
194 if (parentAccountName != null) {
195 parentAccount = ensureAccountExists(parentAccountName, map,
199 parentAccount = null;
201 lastAccount = new LedgerAccount(profile, accName, parentAccount);
203 list.add(lastAccount);
204 map.put(accName, lastAccount);
206 state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
207 L("→ expecting account amount");
211 case EXPECTING_ACCOUNT_AMOUNT:
212 m = reAccountValue.matcher(line);
213 boolean match_found = false;
218 String value = Objects.requireNonNull(m.group(1));
219 String currency = m.group(2);
220 if (currency == null)
224 Matcher tmpM = reDecimalComma.matcher(value);
226 value = value.replace(".", "");
227 value = value.replace(',', '.');
230 tmpM = reDecimalPoint.matcher(value);
232 value = value.replace(",", "");
233 value = value.replace(" ", "");
236 L("curr=" + currency + ", value=" + value);
237 final float val = Float.parseFloat(value);
238 lastAccount.addAmount(val, currency);
239 for (LedgerAccount syn : syntheticAccounts) {
240 L(String.format(Locale.ENGLISH, "propagating %s %1.2f to %s",
241 currency, val, syn.getName()));
242 syn.addAmount(val, currency);
247 syntheticAccounts.clear();
248 state = ParserState.EXPECTING_ACCOUNT;
249 L("→ expecting account");
254 case EXPECTING_TRANSACTION:
255 if (!line.isEmpty() && (line.charAt(0) == ' '))
257 m = reTransactionStart.matcher(line);
259 transactionId = Integer.parseInt(Objects.requireNonNull(m.group(1)));
260 state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
261 L(String.format(Locale.ENGLISH,
262 "found transaction %d → expecting description", transactionId));
263 progress.setProgress(++processedTransactionCount);
264 if (maxTransactionId < transactionId)
265 maxTransactionId = transactionId;
266 if ((progress.isIndeterminate()) ||
267 (progress.getTotal() < transactionId))
268 progress.setTotal(transactionId);
269 publishProgress(progress);
271 m = reEnd.matcher(line);
273 L("--- transaction value complete ---");
278 case EXPECTING_TRANSACTION_DESCRIPTION:
279 if (!line.isEmpty() && (line.charAt(0) == ' '))
281 m = reTransactionDescription.matcher(line);
283 if (transactionId == 0)
284 throw new TransactionParserException(
285 "Transaction Id is 0 while expecting " + "description");
287 String date = Objects.requireNonNull(m.group(1));
289 int equalsIndex = date.indexOf('=');
290 if (equalsIndex >= 0)
291 date = date.substring(equalsIndex + 1);
293 new LedgerTransaction(transactionId, date, m.group(2));
295 catch (ParseException e) {
297 return String.format("Error parsing date '%s'", date);
299 state = ParserState.EXPECTING_TRANSACTION_DETAILS;
300 L(String.format(Locale.ENGLISH,
301 "transaction %d created for %s (%s) →" + " expecting details",
302 transactionId, date, m.group(2)));
306 case EXPECTING_TRANSACTION_DETAILS:
307 if (line.isEmpty()) {
308 // transaction data collected
310 transaction.finishLoading();
311 transactions.add(transaction);
313 state = ParserState.EXPECTING_TRANSACTION;
314 L(String.format("transaction %s parsed → expecting transaction",
315 transaction.getId()));
317 // sounds like a good idea, but transaction-1 may not be the first one chronologically
318 // for example, when you add the initial seeding transaction after entering some others
319 // if (transactionId == 1) {
320 // L("This was the initial transaction.
327 LedgerTransactionAccount lta = parseTransactionAccountLine(line);
329 transaction.addAccount(lta);
330 L(String.format(Locale.ENGLISH, "%d: %s = %s", transaction.getId(),
331 lta.getAccountName(), lta.getAmount()));
334 throw new IllegalStateException(
335 String.format("Can't parse transaction %d details: %s",
336 transactionId, line));
340 throw new RuntimeException(
341 String.format("Unknown parser updating %s", state.name()));
347 profile.setAndStoreAccountAndTransactionListFromWeb(list, transactions);
353 LedgerAccount ensureAccountExists(String accountName, HashMap<String, LedgerAccount> map,
354 ArrayList<LedgerAccount> createdAccounts) {
355 LedgerAccount acc = map.get(accountName);
360 String parentName = LedgerAccount.extractParentName(accountName);
361 LedgerAccount parentAccount;
362 if (parentName != null) {
363 parentAccount = ensureAccountExists(parentName, map, createdAccounts);
366 parentAccount = null;
369 acc = new LedgerAccount(profile, accountName, parentAccount);
370 createdAccounts.add(acc);
373 private boolean retrieveAccountList() throws IOException, HTTPException {
374 HttpURLConnection http = NetworkUtil.prepareConnection(profile, "accounts");
375 http.setAllowUserInteraction(false);
376 switch (http.getResponseCode()) {
382 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
384 publishProgress(Progress.indeterminate());
385 SQLiteDatabase db = App.getDatabase();
386 ArrayList<LedgerAccount> list = new ArrayList<>();
387 HashMap<String, LedgerAccount> map = new HashMap<>();
388 HashMap<String, LedgerAccount> currentMap = new HashMap<>();
389 for (LedgerAccount acc : Objects.requireNonNull(profile.getAllAccounts()))
390 currentMap.put(acc.getName(), acc);
391 try (InputStream resp = http.getInputStream()) {
392 if (http.getResponseCode() != 200)
393 throw new IOException(String.format("HTTP error %d", http.getResponseCode()));
395 AccountListParser parser = new AccountListParser(resp);
399 ParsedLedgerAccount parsedAccount = parser.nextAccount();
400 if (parsedAccount == null) {
404 final String accName = parsedAccount.getAname();
405 LedgerAccount acc = map.get(accName);
407 throw new RuntimeException(
408 String.format("Account '%s' already present", acc.getName()));
409 String parentName = LedgerAccount.extractParentName(accName);
410 ArrayList<LedgerAccount> createdParents = new ArrayList<>();
411 LedgerAccount parent;
412 if (parentName == null) {
416 parent = ensureAccountExists(parentName, map, createdParents);
417 parent.setHasSubAccounts(true);
419 acc = new LedgerAccount(profile, accName, parent);
421 map.put(accName, acc);
423 String lastCurrency = null;
424 float lastCurrencyAmount = 0;
425 for (ParsedBalance b : parsedAccount.getAibalance()) {
427 final String currency = b.getAcommodity();
428 final float amount = b.getAquantity()
430 if (currency.equals(lastCurrency)) {
431 lastCurrencyAmount += amount;
434 if (lastCurrency != null) {
435 acc.addAmount(lastCurrencyAmount, lastCurrency);
437 lastCurrency = currency;
438 lastCurrencyAmount = amount;
441 if (lastCurrency != null) {
442 acc.addAmount(lastCurrencyAmount, lastCurrency);
444 for (LedgerAccount p : createdParents)
445 acc.propagateAmountsTo(p);
450 // the current account tree may have changed, update the new-to be tree to match
451 for (LedgerAccount acc : list) {
452 LedgerAccount prevData = currentMap.get(acc.getName());
453 if (prevData != null) {
454 acc.setExpanded(prevData.isExpanded());
455 acc.setAmountsExpanded(prevData.amountsExpanded());
459 profile.setAndStoreAccountListFromWeb(list);
462 private boolean retrieveTransactionList() throws IOException, ParseException, HTTPException {
463 Progress progress = new Progress();
464 int maxTransactionId = Data.transactions.size();
466 HttpURLConnection http = NetworkUtil.prepareConnection(profile, "transactions");
467 http.setAllowUserInteraction(false);
468 publishProgress(progress);
469 switch (http.getResponseCode()) {
475 throw new HTTPException(http.getResponseCode(), http.getResponseMessage());
477 try (InputStream resp = http.getInputStream()) {
479 ArrayList<LedgerTransaction> trList = new ArrayList<>();
481 TransactionListParser parser = new TransactionListParser(resp);
483 int processedTransactionCount = 0;
487 ParsedLedgerTransaction parsedTransaction = parser.nextTransaction();
489 if (parsedTransaction == null)
492 LedgerTransaction transaction = parsedTransaction.asLedgerTransaction();
493 trList.add(transaction);
495 progress.setProgress(++processedTransactionCount);
496 publishProgress(progress);
500 profile.setAndStoreTransactionList(trList);
506 @SuppressLint("DefaultLocale")
508 protected String doInBackground(Void... params) {
509 Data.backgroundTaskStarted();
511 if (!retrieveAccountList() || !retrieveTransactionList())
512 return retrieveTransactionListLegacy();
515 catch (MalformedURLException e) {
517 return "Invalid server URL";
519 catch (HTTPException e) {
521 return String.format("HTTP error %d: %s", e.getResponseCode(), e.getResponseMessage());
523 catch (IOException e) {
525 return e.getLocalizedMessage();
527 catch (ParseException e) {
529 return "Network error";
531 catch (OperationCanceledException e) {
533 return "Operation cancelled";
536 Data.backgroundTaskFinished();
539 private void throwIfCancelled() {
541 throw new OperationCanceledException(null);
543 private enum ParserState {
544 EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_TRANSACTION,
545 EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
548 public enum ProgressState {STARTING, RUNNING, FINISHED}
550 public static class Progress {
551 private int progress;
553 private ProgressState state = ProgressState.RUNNING;
554 private String error = null;
555 private boolean indeterminate;
557 indeterminate = true;
559 Progress(int progress, int total) {
560 this.indeterminate = false;
561 this.progress = progress;
564 public static Progress indeterminate() {
565 return new Progress();
567 public static Progress finished(String error) {
568 Progress p = new Progress();
569 p.setState(ProgressState.FINISHED);
573 public int getProgress() {
574 ensureState(ProgressState.RUNNING);
577 protected void setProgress(int progress) {
578 this.progress = progress;
579 this.state = ProgressState.RUNNING;
581 public int getTotal() {
582 ensureState(ProgressState.RUNNING);
585 protected void setTotal(int total) {
587 state = ProgressState.RUNNING;
588 indeterminate = total == -1;
590 private void ensureState(ProgressState wanted) {
592 throw new IllegalStateException(
593 String.format("Bad state: %s, expected %s", state, wanted));
595 public ProgressState getState() {
598 public void setState(ProgressState state) {
601 public String getError() {
602 ensureState(ProgressState.FINISHED);
605 public void setError(String error) {
607 state = ProgressState.FINISHED;
609 public boolean isIndeterminate() {
610 return indeterminate;
612 public void setIndeterminate(boolean indeterminate) {
613 this.indeterminate = indeterminate;
617 private static class TransactionParserException extends IllegalStateException {
618 TransactionParserException(String message) {