Merge pull request #137 from markferry/awardwallet-v0.2

awardwallet v0.2

Author: tarioch

docs/importers.rst

@@ -447,48 +447,65 @@ Import PDF from `radicant <https://radicant.com/>`__
 AwardWallet
 ------------------------------
 
-Import from `AwardWallet <https://awardwallet.com/>`__ using their `Account Access API <https://awardwallet.com/api/account>`__.
+Import from `AwardWallet <https://awardwallet.com/>`__ using their `Account
+Access API <https://awardwallet.com/api/account>`__.
 
-As of 2025 AwardWallet integrates over 460 airline, hotel, shopping and other loyalty programmes.
+As of 2025 AwardWallet integrates over 460 airline, hotel, shopping and other
+loyalty programmes. Many programmes do not support retrieval of transactions,
+only balances.
 
-Follow the instructions in the `API documentation <https://awardwallet.com/api/account#introduction>`__ to register for a free Business account and create an API key.
+1.  Update your personal AwardWallet account.
+    The importer can only retrieve data that has already been synced to your AwardWallet account.
 
-The API key is restricted to the **allowed IP addresses** you specify in the Business interface API Settings.
+2.  Follow the instructions in the `API documentation
+    <https://awardwallet.com/api/account#introduction>`__ to register for a
+    free Business account and create an API key.
 
-Link and authorize personal accounts using the included ``awardwallet-conf`` CLI tool:
+    The API key is restricted to the **allowed IP addresses** you specify in
+    the Business interface API Settings.
 
-.. code-block:: console
+3.  Link and authorize personal accounts using the included ``awardwallet-conf``
+    CLI tool:
 
-  awardwallet-conf --api-key YOUR_API_KEY get_link_url
+    .. code-block:: console
 
+      awardwallet-conf --api-key YOUR_API_KEY get_link_url
 
-Generate a config file for all linked users called (or ending with) ``awardwallet.yaml`` in your import location (e.g. download folder) and edit it to your needs.
-Note that not all providers support retrieval of transaction history.
+    Manage access to your personal account under `Manage Users
+    <https://awardwallet.com/user/connections>`__ in the personal web
+    interface.
 
-.. code-block:: console
+4.  Generate a config file for all linked users ending with
+    ``awardwallet.yaml`` in your import location (e.g. download folder) and
+    edit it to your needs.
 
-  awardwallet-conf --api-key YOUR_API_KEY generate > awardwallet.yaml
+    .. code-block:: console
 
-Example configuration file:
+      awardwallet-conf --api-key YOUR_API_KEY generate > awardwallet.yaml
 
-.. code-block:: yaml
+    If you have multiple accounts using the same points currency (e.g. ``AVIOS``
+    used by BA, Iberia, ...) create sub-accounts for each so that ``balance``
+    directives will work.
 
-  api_key: YOUR_API_KEY
-  users:
-    12345:
-      name: John Smith
-      all_history: false
-      accounts:
-        7654321:
-          provider: "British Airways Club"
-          account: Assets:Current:Points
-          currency: AVIOS
+    Example configuration file:
 
+    .. code-block:: yaml
 
-Finally, initialize the importer:
+      api_key: YOUR_API_KEY
+      users:
+        12345:
+          name: John Smith
+          all_history: false     # only the last 10 txns per account
+          accounts:
+            7654321:
+              provider: "British Airways Club"
+              account: Assets:Current:Points
+              currency: AVIOS
 
-.. code-block:: python
+5.  Finally, initialize the importer:
+
+    .. code-block:: python
 
-  from tariochbctools.importers.awardwalletimp import importer as awimp
+      from tariochbctools.importers.awardwalletimp import importer as awimp
 
-  CONFIG = [awimp.Importer()]
+      CONFIG = [awimp.Importer()]

setup.cfg

@@ -37,7 +37,7 @@ package_dir =
 # install_requires = numpy; scipy
 install_requires =
     importlib-metadata; python_version<"3.8"
-    awardwallet
+    awardwallet>=0.2
     beancount>=3
     beangulp
     beanprice

src/tariochbctools/importers/awardwalletimp/config.py

@@ -5,7 +5,7 @@
 
 import yaml
 from awardwallet import AwardWalletClient
-from awardwallet.api import AccessLevel
+from awardwallet.client import AccessLevel
 
 
 def get_link_url(client):
@@ -33,19 +33,18 @@ def generate(client):
     connected_users = client.list_connected_users()
 
     for user in connected_users:
-        user_id = user["userId"]
-        user_details = client.get_connected_user_details(user_id)
+        user_details = client.get_connected_user_details(user.user_id)
         account_config = {}
 
-        for account in user_details.get("accounts", []):
-            account_config[account["accountId"]] = {
-                "provider": account["displayName"],
+        for account in user_details.accounts:
+            account_config[account.account_id] = {
+                "provider": account.display_name,
                 "account": "Assets:Current:Points",  # Placeholder, user should adjust
                 "currency": "POINTS",
             }
 
-        config["users"][user_id] = {
-            "name": user["userName"],
+        config["users"][user.user_id] = {
+            "name": user.user_name,
             "all_history": False,
             "accounts": account_config,
         }

src/tariochbctools/importers/awardwalletimp/importer.py

@@ -1,11 +1,13 @@
+import datetime
 import logging
+import re
+from operator import attrgetter
 from os import path
-from typing import Any
 
 import beangulp
 import dateutil.parser
 import yaml
-from awardwallet import AwardWalletClient
+from awardwallet import AwardWalletClient, model
 from beancount.core import amount, data
 from beancount.core.number import D
 
@@ -46,27 +48,33 @@ def extract(self, filepath: str, existing: data.Entries = None) -> data.Entries:
         return entries
 
     def _extract_user_history(
-        self, user: dict, user_details: dict
+        self, user: dict, user_details: model.GetConnectedUserDetailsResponse
     ) -> list[data.Transaction]:
         """
         User history is limited to the last 10 history elements per account
         """
         entries = []
-        for account in user_details["accounts"]:
-            account_id = account["accountId"]
-
-            if account_id in user["accounts"]:
-                logging.info("Extracting account ID %s", account_id)
-                account_config = user["accounts"][account_id]
+        for account in user_details.accounts:
+            if account.account_id in user["accounts"]:
+                logging.info("Extracting account ID %s", account.account_id)
+                account_config = user["accounts"][account.account_id]
 
                 entries.extend(
                     self._extract_transactions(
-                        account["history"], account_config, account_id
+                        account.history, account_config, account.account_id
                     )
                 )
+
+                # we fudge the date by using the latest txn (across *all* accounts)
+                latest_txn = max(entries, key=attrgetter("date"), default=None)
+                entries.extend(
+                    self._extract_balance(account, account_config, latest_txn)
+                )
             else:
                 logging.warning(
-                    "Ignoring account ID %s: %s", account_id, account["displayName"]
+                    "Ignoring account ID %s: %s",
+                    account.account_id,
+                    account.display_name,
                 )
         return entries
 
@@ -76,13 +84,15 @@ def _extract_account_history(
         entries = []
         for account_id, account_config in user["accounts"].items():
             logging.info("Extracting account ID %s", account_id)
-            account = client.get_account_details(account_id)["account"]
+            account = client.get_account_details(account_id).account
 
             entries.extend(
-                self._extract_transactions(
-                    account["history"], account_config, account_id
-                )
+                self._extract_transactions(account.history, account_config, account_id)
             )
+
+            # we fudge the date by using the latest txn (across *all* accounts)
+            latest_txn = max(entries, key=attrgetter("date"), default=None)
+            entries.extend(self._extract_balance(account, account_config, latest_txn))
         return entries
 
     def _extract_transactions(
@@ -109,7 +119,7 @@ def _extract_transactions(
 
     def _extract_transaction(
         self,
-        trx: dict[str, Any],
+        trx: model.HistoryItem,
         local_account: data.Account,
         currency: str,
         account_id: str,
@@ -121,20 +131,23 @@ def _extract_transaction(
         trx_description = None
         trx_amount = None
 
-        for f in trx.get("fields", []):
-            if f["code"] == "PostingDate":
-                trx_date = dateutil.parser.parse(f["value"]["value"]).date()
-            if f["code"] == "Description":
-                trx_description = f["value"]["value"].replace("\n", " ")
-            if f["code"] == "Miles":
-                trx_amount = D(f["value"]["value"])
-            if f["code"] == "Info":
-                name = f["name"].lower().replace(" ", "-")
-                metakv[name] = f["value"]["value"].replace("\n", " ")
+        for f in trx.fields:
+            if f.code == "PostingDate":
+                trx_date = dateutil.parser.parse(f.value.value).date()
+            if f.code == "Description":
+                trx_description = f.value.value.replace("\n", " ")
+            if f.code == "Miles":
+                trx_amount = D(f.value.value)
+            if f.code == "Info":
+                name = re.sub(r"\W", "-", f.name).lower()
+                metakv[name] = f.value.value.replace("\n", " ")
 
         assert trx_date
         assert trx_description
-        assert trx_amount is not None, f"No amount in trx: {trx}"
+
+        if trx_amount is None:
+            logging.warning("Skipping transaction with no amount: %s", trx)
+            return []
 
         meta = data.new_metadata("", 0, metakv)
         entry = data.Transaction(
@@ -158,3 +171,36 @@ def _extract_transaction(
         )
         entries.append(entry)
         return entries
+
+    def _extract_balance(
+        self,
+        account: model.Account,
+        account_config: dict,
+        latest_txn: data.Transaction | None,
+    ) -> list[data.Transaction]:
+        local_account = account_config["account"]
+        currency = account_config["currency"]
+        balance = amount.Amount(D(account.balance_raw), currency)
+        metakv = {"account-id": str(account.account_id)}
+
+        optional_date = account.last_change_date or account.last_retrieve_date
+        if optional_date:
+            date = optional_date.date()
+        elif latest_txn:
+            date = latest_txn.date
+        else:
+            logging.warning(
+                "No date information available for balance of account %s, using today",
+                account.account_id,
+            )
+            date = datetime.date.today()
+
+        entry = data.Balance(
+            data.new_metadata("", 0, metakv),
+            date + datetime.timedelta(days=1),
+            local_account,
+            balance,
+            None,
+            None,
+        )
+        return [entry]