]> git.ktnx.net Git - mobile-ledger.git/blob - app/src/main/java/net/ktnx/mobileledger/async/RetrieveTransactionsTask.java
fix crash when profile has no value yet
[mobile-ledger.git] / app / src / main / java / net / ktnx / mobileledger / async / RetrieveTransactionsTask.java
1 /*
2  * Copyright © 2019 Damyan Ivanov.
3  * This file is part of MoLe.
4  * MoLe is free software: you can distribute it and/or modify it
5  * under the term of the GNU General Public License as published by
6  * the Free Software Foundation, either version 3 of the License, or
7  * (at your opinion), any later version.
8  *
9  * MoLe is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  * GNU General Public License terms for details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with MoLe. If not, see <https://www.gnu.org/licenses/>.
16  */
17
18 package net.ktnx.mobileledger.async;
19
20 import android.annotation.SuppressLint;
21 import android.database.sqlite.SQLiteDatabase;
22 import android.os.AsyncTask;
23 import android.os.OperationCanceledException;
24 import android.util.Log;
25
26 import net.ktnx.mobileledger.model.Data;
27 import net.ktnx.mobileledger.model.LedgerAccount;
28 import net.ktnx.mobileledger.model.LedgerTransaction;
29 import net.ktnx.mobileledger.model.LedgerTransactionAccount;
30 import net.ktnx.mobileledger.model.MobileLedgerProfile;
31 import net.ktnx.mobileledger.ui.activity.MainActivity;
32 import net.ktnx.mobileledger.ui.transaction_list.TransactionListViewModel;
33 import net.ktnx.mobileledger.utils.MLDB;
34 import net.ktnx.mobileledger.utils.NetworkUtil;
35
36 import java.io.BufferedReader;
37 import java.io.FileNotFoundException;
38 import java.io.IOException;
39 import java.io.InputStream;
40 import java.io.InputStreamReader;
41 import java.lang.ref.WeakReference;
42 import java.net.HttpURLConnection;
43 import java.net.MalformedURLException;
44 import java.net.URLDecoder;
45 import java.nio.charset.StandardCharsets;
46 import java.text.ParseException;
47 import java.util.ArrayList;
48 import java.util.Date;
49 import java.util.HashMap;
50 import java.util.Stack;
51 import java.util.regex.Matcher;
52 import java.util.regex.Pattern;
53
54
55 public class RetrieveTransactionsTask
56         extends AsyncTask<Void, RetrieveTransactionsTask.Progress, String> {
57     private static final int MATCHING_TRANSACTIONS_LIMIT = 50;
58     private static final Pattern reComment = Pattern.compile("^\\s*;");
59     private static final Pattern reTransactionStart = Pattern.compile("<tr class=\"title\" " +
60                                                                       "id=\"transaction-(\\d+)\"><td class=\"date\"[^\"]*>([\\d.-]+)</td>");
61     private static final Pattern reTransactionDescription =
62             Pattern.compile("<tr class=\"posting\" title=\"(\\S+)\\s(.+)");
63     private static final Pattern reTransactionDetails =
64             Pattern.compile("^\\s+(\\S[\\S\\s]+\\S)\\s\\s+([-+]?\\d[\\d,.]*)(?:\\s+(\\S+)$)?");
65     private static final Pattern reEnd = Pattern.compile("\\bid=\"addmodal\"");
66     private WeakReference<MainActivity> contextRef;
67     private int error;
68     // %3A is '='
69     private Pattern reAccountName = Pattern.compile("/register\\?q=inacct%3A([a-zA-Z0-9%]+)\"");
70     private Pattern reAccountValue = Pattern.compile(
71             "<span class=\"[^\"]*\\bamount\\b[^\"]*\">\\s*([-+]?[\\d.,]+)(?:\\s+(\\S+))?</span>");
72     public RetrieveTransactionsTask(WeakReference<MainActivity> contextRef) {
73         this.contextRef = contextRef;
74     }
75     private static void L(String msg) {
76         Log.d("transaction-parser", msg);
77     }
78     @Override
79     protected void onProgressUpdate(Progress... values) {
80         super.onProgressUpdate(values);
81         MainActivity context = getContext();
82         if (context == null) return;
83         context.onRetrieveProgress(values[0]);
84     }
85     @Override
86     protected void onPreExecute() {
87         super.onPreExecute();
88         MainActivity context = getContext();
89         if (context == null) return;
90         context.onRetrieveStart();
91     }
92     @Override
93     protected void onPostExecute(String error) {
94         super.onPostExecute(error);
95         MainActivity context = getContext();
96         if (context == null) return;
97         context.onRetrieveDone(error);
98     }
99     @Override
100     protected void onCancelled() {
101         super.onCancelled();
102         MainActivity context = getContext();
103         if (context == null) return;
104         context.onRetrieveDone(null);
105     }
106     @SuppressLint("DefaultLocale")
107     @Override
108     protected String doInBackground(Void... params) {
109         MobileLedgerProfile profile = Data.profile.get();
110         Progress progress = new Progress();
111         int maxTransactionId = Progress.INDETERMINATE;
112         ArrayList<LedgerAccount> accountList = new ArrayList<>();
113         HashMap<String, Void> accountNames = new HashMap<>();
114         LedgerAccount lastAccount = null;
115         boolean onlyStarred = Data.optShowOnlyStarred.get();
116         Data.backgroundTaskCount.incrementAndGet();
117         try {
118             HttpURLConnection http = NetworkUtil.prepareConnection("journal");
119             http.setAllowUserInteraction(false);
120             publishProgress(progress);
121             MainActivity ctx = getContext();
122             if (ctx == null) return null;
123             try (SQLiteDatabase db = MLDB.getWritableDatabase()) {
124                 try (InputStream resp = http.getInputStream()) {
125                     if (http.getResponseCode() != 200) throw new IOException(
126                             String.format("HTTP error %d", http.getResponseCode()));
127                     db.beginTransaction();
128                     try {
129                         db.execSQL("UPDATE transactions set keep=0");
130                         db.execSQL("update account_values set keep=0;");
131                         db.execSQL("update accounts set keep=0;");
132
133                         ParserState state = ParserState.EXPECTING_ACCOUNT;
134                         String line;
135                         BufferedReader buf = new BufferedReader(
136                                 new InputStreamReader(resp, StandardCharsets.UTF_8));
137
138                         int processedTransactionCount = 0;
139                         int transactionId = 0;
140                         int matchedTransactionsCount = 0;
141                         LedgerTransaction transaction = null;
142                         LINES:
143                         while ((line = buf.readLine()) != null) {
144                             throwIfCancelled();
145                             Matcher m;
146                             m = reComment.matcher(line);
147                             if (m.find()) {
148                                 // TODO: comments are ignored for now
149                                 Log.v("transaction-parser", "Ignoring comment");
150                                 continue;
151                             }
152                             //L(String.format("State is %d", updating));
153                             switch (state) {
154                                 case EXPECTING_ACCOUNT:
155                                     if (line.equals("<h2>General Journal</h2>")) {
156                                         state = ParserState.EXPECTING_TRANSACTION;
157                                         L("→ expecting transaction");
158                                         Data.accounts.set(accountList);
159                                         continue;
160                                     }
161                                     m = reAccountName.matcher(line);
162                                     if (m.find()) {
163                                         String acct_encoded = m.group(1);
164                                         String acct_name = URLDecoder.decode(acct_encoded, "UTF-8");
165                                         acct_name = acct_name.replace("\"", "");
166                                         L(String.format("found account: %s", acct_name));
167
168                                         lastAccount = profile.loadAccount(acct_name);
169                                         if (lastAccount == null) {
170                                             lastAccount = new LedgerAccount(acct_name);
171                                             profile.storeAccount(lastAccount);
172                                         }
173
174                                         // make sure the parent account(s) are present,
175                                         // synthesising them if necessary
176                                         String parentName = lastAccount.getParentName();
177                                         if (parentName != null) {
178                                             Stack<String> toAppend = new Stack<>();
179                                             while (parentName != null) {
180                                                 if (accountNames.containsKey(parentName)) break;
181                                                 toAppend.push(parentName);
182                                                 parentName = new LedgerAccount(parentName)
183                                                         .getParentName();
184                                             }
185                                             while (!toAppend.isEmpty()) {
186                                                 String aName = toAppend.pop();
187                                                 LedgerAccount acc = new LedgerAccount(aName);
188                                                 acc.setHidden(lastAccount.isHidden());
189                                                 if (!onlyStarred || !acc.isHidden())
190                                                     accountList.add(acc);
191                                                 L(String.format("gap-filling with %s", aName));
192                                                 accountNames.put(aName, null);
193                                                 profile.storeAccount(acc);
194                                             }
195                                         }
196
197                                         if (!onlyStarred || !lastAccount.isHidden())
198                                             accountList.add(lastAccount);
199                                         accountNames.put(acct_name, null);
200
201                                         state = ParserState.EXPECTING_ACCOUNT_AMOUNT;
202                                         L("→ expecting account amount");
203                                     }
204                                     break;
205
206                                 case EXPECTING_ACCOUNT_AMOUNT:
207                                     m = reAccountValue.matcher(line);
208                                     boolean match_found = false;
209                                     while (m.find()) {
210                                         throwIfCancelled();
211
212                                         match_found = true;
213                                         String value = m.group(1);
214                                         String currency = m.group(2);
215                                         if (currency == null) currency = "";
216                                         value = value.replace(',', '.');
217                                         L("curr=" + currency + ", value=" + value);
218                                         profile.storeAccountValue(lastAccount.getName(), currency,
219                                                 Float.valueOf(value));
220                                         lastAccount.addAmount(Float.parseFloat(value), currency);
221                                     }
222
223                                     if (match_found) {
224                                         state = ParserState.EXPECTING_ACCOUNT;
225                                         L("→ expecting account");
226                                     }
227
228                                     break;
229
230                                 case EXPECTING_TRANSACTION:
231                                     if (!line.isEmpty() && (line.charAt(0) == ' ')) continue;
232                                     m = reTransactionStart.matcher(line);
233                                     if (m.find()) {
234                                         transactionId = Integer.valueOf(m.group(1));
235                                         state = ParserState.EXPECTING_TRANSACTION_DESCRIPTION;
236                                         L(String.format(
237                                                 "found transaction %d → expecting description",
238                                                 transactionId));
239                                         progress.setProgress(++processedTransactionCount);
240                                         if (maxTransactionId < transactionId)
241                                             maxTransactionId = transactionId;
242                                         if ((progress.getTotal() == Progress.INDETERMINATE) ||
243                                             (progress.getTotal() < transactionId))
244                                             progress.setTotal(transactionId);
245                                         publishProgress(progress);
246                                     }
247                                     m = reEnd.matcher(line);
248                                     if (m.find()) {
249                                         L("--- transaction value complete ---");
250                                         break LINES;
251                                     }
252                                     break;
253
254                                 case EXPECTING_TRANSACTION_DESCRIPTION:
255                                     if (!line.isEmpty() && (line.charAt(0) == ' ')) continue;
256                                     m = reTransactionDescription.matcher(line);
257                                     if (m.find()) {
258                                         if (transactionId == 0)
259                                             throw new TransactionParserException(
260                                                     "Transaction Id is 0 while expecting " +
261                                                     "description");
262
263                                         String date = m.group(1);
264                                         try {
265                                             int equalsIndex = date.indexOf('=');
266                                             if (equalsIndex >= 0)
267                                                 date = date.substring(equalsIndex + 1);
268                                             transaction = new LedgerTransaction(transactionId, date,
269                                                     m.group(2));
270                                         }
271                                         catch (ParseException e) {
272                                             e.printStackTrace();
273                                             return String.format("Error parsing date '%s'", date);
274                                         }
275                                         state = ParserState.EXPECTING_TRANSACTION_DETAILS;
276                                         L(String.format("transaction %d created for %s (%s) →" +
277                                                         " expecting details", transactionId, date,
278                                                 m.group(2)));
279                                     }
280                                     break;
281
282                                 case EXPECTING_TRANSACTION_DETAILS:
283                                     if (line.isEmpty()) {
284                                         // transaction data collected
285                                         if (transaction.existsInDb(db)) {
286                                             db.execSQL("UPDATE transactions SET keep = 1 WHERE " +
287                                                        "profile = ? and id=?",
288                                                     new Object[]{profile.getUuid(),
289                                                                  transaction.getId()
290                                                     });
291                                             matchedTransactionsCount++;
292
293                                             if (matchedTransactionsCount ==
294                                                 MATCHING_TRANSACTIONS_LIMIT)
295                                             {
296                                                 db.execSQL("UPDATE transactions SET keep=1 WHERE " +
297                                                            "profile = ? and id < ?",
298                                                         new Object[]{profile.getUuid(),
299                                                                      transaction.getId()
300                                                         });
301                                                 progress.setTotal(progress.getProgress());
302                                                 publishProgress(progress);
303                                                 break LINES;
304                                             }
305                                         }
306                                         else {
307                                             profile.storeTransaction(transaction);
308                                             matchedTransactionsCount = 0;
309                                             progress.setTotal(maxTransactionId);
310                                         }
311
312                                         state = ParserState.EXPECTING_TRANSACTION;
313                                         L(String.format(
314                                                 "transaction %s saved → expecting transaction",
315                                                 transaction.getId()));
316                                         transaction.finishLoading();
317
318 // sounds like a good idea, but transaction-1 may not be the first one chronologically
319 // for example, when you add the initial seeding transaction after entering some others
320 //                                            if (transactionId == 1) {
321 //                                                L("This was the initial transaction. Terminating " +
322 //                                                  "parser");
323 //                                                break LINES;
324 //                                            }
325                                     }
326                                     else {
327                                         m = reTransactionDetails.matcher(line);
328                                         if (m.find()) {
329                                             String acc_name = m.group(1);
330                                             String amount = m.group(2);
331                                             String currency = m.group(3);
332                                             if (currency == null) currency = "";
333                                             amount = amount.replace(',', '.');
334                                             transaction.addAccount(
335                                                     new LedgerTransactionAccount(acc_name,
336                                                             Float.valueOf(amount), currency));
337                                             L(String.format("%d: %s = %s", transaction.getId(),
338                                                     acc_name, amount));
339                                         }
340                                         else throw new IllegalStateException(String.format(
341                                                 "Can't parse transaction %d " + "details: %s",
342                                                 transactionId, line));
343                                     }
344                                     break;
345                                 default:
346                                     throw new RuntimeException(
347                                             String.format("Unknown parser updating %s",
348                                                     state.name()));
349                             }
350                         }
351
352                         throwIfCancelled();
353
354                         db.execSQL("DELETE FROM transactions WHERE profile=? AND keep = 0",
355                                 new String[]{profile.getUuid()});
356                         db.setTransactionSuccessful();
357
358                         Log.d("db", "Updating transaction value stamp");
359                         Date now = new Date();
360                         profile.setLongOption(MLDB.OPT_LAST_SCRAPE, now.getTime());
361                         Data.lastUpdateDate.set(now);
362                         TransactionListViewModel.scheduleTransactionListReload();
363
364                         return null;
365                     }
366                     finally {
367                         db.endTransaction();
368                     }
369                 }
370             }
371         }
372         catch (MalformedURLException e) {
373             e.printStackTrace();
374             return "Invalid server URL";
375         }
376         catch (FileNotFoundException e) {
377             e.printStackTrace();
378             return "Invalid user name or password";
379         }
380         catch (IOException e) {
381             e.printStackTrace();
382             return "Network error";
383         }
384         catch (OperationCanceledException e) {
385             e.printStackTrace();
386             return "Operation cancelled";
387         }
388         finally {
389             Data.backgroundTaskCount.decrementAndGet();
390         }
391     }
392     private MainActivity getContext() {
393         return contextRef.get();
394     }
395     private void throwIfCancelled() {
396         if (isCancelled()) throw new OperationCanceledException(null);
397     }
398
399     private enum ParserState {
400         EXPECTING_ACCOUNT, EXPECTING_ACCOUNT_AMOUNT, EXPECTING_JOURNAL, EXPECTING_TRANSACTION,
401         EXPECTING_TRANSACTION_DESCRIPTION, EXPECTING_TRANSACTION_DETAILS
402     }
403
404     public class Progress {
405         public static final int INDETERMINATE = -1;
406         private int progress;
407         private int total;
408         Progress() {
409             this(INDETERMINATE, INDETERMINATE);
410         }
411         Progress(int progress, int total) {
412             this.progress = progress;
413             this.total = total;
414         }
415         public int getProgress() {
416             return progress;
417         }
418         protected void setProgress(int progress) {
419             this.progress = progress;
420         }
421         public int getTotal() {
422             return total;
423         }
424         protected void setTotal(int total) {
425             this.total = total;
426         }
427     }
428
429     private class TransactionParserException extends IllegalStateException {
430         TransactionParserException(String message) {
431             super(message);
432         }
433     }
434 }