Merge pull request #265 from Der-Henning/dev

Webhook improvements

DOCKER_README.md

@@ -10,8 +10,8 @@ Readme, source and documentation on [https://github.com/Der-Henning/tgtg](https:
 
 - [`edge`](https://github.com/Der-Henning/tgtg/blob/main/Dockerfile)
 - [`edge-alpine`](https://github.com/Der-Henning/tgtg/blob/main/Dockerfile.alpine)
-- [`v1`, `v1.14`, `v1.14.9`, `latest`](https://github.com/Der-Henning/tgtg/blob/v1.14.9/Dockerfile)
-- [`v1-alpine`, `v1.14-alpine`, `v1.14.9-alpine`, `latest-alpine`](https://github.com/Der-Henning/tgtg/blob/v1.14.9/Dockerfile.alpine)
+- [`v1`, `v1.14`, `v1.14.10`, `latest`](https://github.com/Der-Henning/tgtg/blob/v1.14.10/Dockerfile)
+- [`v1-alpine`, `v1.14-alpine`, `v1.14.10-alpine`, `latest-alpine`](https://github.com/Der-Henning/tgtg/blob/v1.14.10/Dockerfile.alpine)
 
 # Quick Start
 

docker-compose.yml

@@ -67,6 +67,7 @@ services:
       - WEBHOOK_METHOD=POST                           ## Request Method
       - WEBHOOK_BODY={}                               ## Data to send, can contain $${{variable}}
       - WEBHOOK_TYPE=application/json                 ## Content-Type for header, default: text/plain
+      - WEBHOOK_HEADERS={}                            ## Additional headers
       - WEBHOOK_TIMEOUT=60                            ## Request Timeout
     volumes:
       - tokens:/tokens           ## volume to save TGTG credentials to reuse on next start up and avoid login mail

src/_version.py

@@ -1,6 +1,6 @@
 __title__ = "TGTG Scanner"
 __description__ = "Provides notifications for TGTG magic bags"
-__version__ = "1.14.9"
+__version__ = "1.14.10"
 __author__ = "Henning Merklinger"
 __author_email__ = "[email protected]"
 __license__ = "GPL"

src/config.sample.ini

@@ -95,5 +95,6 @@ URL =
 Method = POST
 body =
 type = text/plain
+headers =
 timeout = 60
 # cron =

src/models/config.py

@@ -1,5 +1,6 @@
 import codecs
 import configparser
+import json
 import logging
 from io import TextIOWrapper
 from os import environ
@@ -75,6 +76,7 @@
         'method': 'POST',
         'body': '',
         'type': '',
+        'headers': {},
         'timeout': 60,
         'cron': Cron('* * * * *')
     },
@@ -207,6 +209,14 @@ def _ini_get_array(self, config: configparser.ConfigParser,
                 arr = [self._decode(val.strip()) for val in value.split(',')]
                 self._setattr(attr, arr)
 
+    def _ini_get_dict(self, config: configparser.ConfigParser,
+                      section: str, key: str, attr: str) -> None:
+        if section in config:
+            value = config[section].get(key, None)
+            if value:
+                dic = json.loads(value)
+                self._setattr(attr, dic)
+
     def _ini_get_cron(self, config: configparser.ConfigParser,
                       section: str, key: str, attr: str) -> None:
         if section in config:
@@ -280,6 +290,7 @@ def _read_ini(self) -> None:
             self._ini_get(config, "WEBHOOK", "Method", "webhook.method")
             self._ini_get(config, "WEBHOOK", "body", "webhook.body")
             self._ini_get(config, "WEBHOOK", "type", "webhook.type")
+            self._ini_get_dict(config, "WEBHOOK", "headers", "webhook.headers")
             self._ini_get_int(config, "WEBHOOK", "timeout", "webhook.timeout")
             self._ini_get_cron(config, "WEBHOOK", "cron", "webhook.cron")
 
@@ -318,6 +329,12 @@ def _env_get_array(self, key: str, attr: str) -> None:
             arr = [self._decode(val.strip()) for val in value.split(',')]
             self._setattr(attr, arr)
 
+    def _env_get_dict(self, key: str, attr: str) -> None:
+        value = environ.get(key, None)
+        if value:
+            dic = json.loads(value)
+            self._setattr(attr, dic)
+
     def _env_get_cron(self, key: str, attr: str) -> None:
         value = environ.get(key, None)
         if value is not None:
@@ -381,6 +398,7 @@ def _read_env(self) -> None:
             self._env_get("WEBHOOK_METHOD", "webhook.method")
             self._env_get("WEBHOOK_BODY", "webhook.body")
             self._env_get("WEBHOOK_TYPE", "webhook.type")
+            self._env_get_dict("WEBHOOK_HEADERS", "webhook.headers")
             self._env_get_int("WEBHOOK_TIMEOUT", "webhook.timeout")
             self._env_get_cron("WEBHOOK_CRON", "webhook.cron")
 

src/notifiers/base.py

@@ -13,5 +13,5 @@ def __init__(self, config: Config):
     def send(self, item: Item) -> None:
         """Send Item information"""
 
-    def stop(self):
+    def stop(self) -> None:
         """Stop notifier"""

src/notifiers/ifttt.py

@@ -21,6 +21,7 @@ def __init__(self, config: Config):
         self.body = config.ifttt.get("body")
         self.cron = config.ifttt.get("cron")
         self.timeout = config.ifttt.get("timeout")
+        self.headers = {}
         self.method = "POST"
         self.url = (f"https://maker.ifttt.com/trigger/"
                     f"{self.event}/with/key/{self.key}")

src/notifiers/webhook.py

@@ -1,3 +1,4 @@
+import json
 import logging
 
 import requests
@@ -18,6 +19,7 @@ def __init__(self, config: Config):
         self.url = config.webhook.get("url")
         self.body = config.webhook.get("body")
         self.type = config.webhook.get("type")
+        self.headers = config.webhook.get("headers", {})
         self.timeout = config.webhook.get("timeout", 60)
         self.cron = config.webhook.get("cron")
         if self.enabled and (not self.method or not self.url):
@@ -36,15 +38,14 @@ def send(self, item: Item) -> None:
             url = item.unmask(self.url)
             log.debug("Webhook url: %s", url)
             body = None
-            headers = {
-                "Content-Type": self.type
-            }
+            headers = self.headers
+            if self.type:
+                headers["Content-Type"] = self.type
             if self.body:
-                body = item.unmask(self.body).replace(
-                    '\n', '\\n').encode(encoding='UTF-8',
-                                        errors='replace')
-                headers["Content-Length"] = str(len(body))
-                log.debug("Webhook body: %s", body)
+                body = item.unmask(self.body)
+                if self.type and 'json' in self.type:
+                    body = json.dumps(json.loads(body.replace('\n', '\\n')))
+                log.debug("Webhook body: '%s'", body)
             log.debug("Webhook headers: %s", headers)
             res = requests.request(method=self.method, url=url,
                                    timeout=self.timeout, data=body,

src/scanner.py

@@ -5,6 +5,7 @@
 from typing import List, NoReturn
 
 from models import Config, Item, Metrics
+from models.errors import TgtgAPIError
 from notifiers import Notifiers
 from tgtg import TgtgClient
 
@@ -59,21 +60,22 @@ def _job(self) -> None:
         """
         Job iterates over all monitored items
         """
+        items = []
         for item_id in self.item_ids:
             try:
                 if item_id != "":
-                    item = Item(self.tgtg_client.get_item(item_id))
-                    self._check_item(item)
-            except Exception:
-                log.error("itemID %s Error! - %s", item_id, sys.exc_info())
-        for item in self._get_favorites():
-            try:
-                self._check_item(item)
-            except Exception:
-                log.error("check item error! - %s", sys.exc_info())
+                    items.append(Item(self.tgtg_client.get_item(item_id)))
+            except TgtgAPIError as err:
+                log.error(err)
+        items += self._get_favorites()
+        for item in items:
+            self._check_item(item)
+
         log.debug("new State: %s", self.amounts)
+
         if len(self.amounts) == 0:
             log.warning("No items in observation! Did you add any favorites?")
+
         self.config.save_tokens(
             self.tgtg_client.access_token,
             self.tgtg_client.refresh_token,
@@ -85,49 +87,33 @@ def _get_favorites(self) -> list[Item]:
         """
         Get favorites as list of Items
         """
-        items = []
-        page = 1
-        page_size = 100
-        error_count = 0
-        while error_count < 5:
-            try:
-                new_items = self.tgtg_client.get_items(
-                    favorites_only=True, page_size=page_size, page=page
-                )
-                items += new_items
-                if len(new_items) < page_size:
-                    break
-                page += 1
-            except Exception:
-                log.error("get item error! - %s", sys.exc_info())
-                error_count += 1
-                self.metrics.get_favorites_errors.inc()
+        try:
+            items = self.get_favorites()
+        except TgtgAPIError as err:
+            log.error(err)
+            return []
         return [Item(item) for item in items]
 
     def _check_item(self, item: Item) -> None:
         """
         Checks if the available item amount raised from zero to something
         and triggers notifications.
         """
-        try:
-            if (
-                self.amounts[item.item_id] == 0
-                and item.items_available > self.amounts[item.item_id]
-            ):
-                self._send_messages(item)
-                self.metrics.send_notifications.labels(
-                    item.item_id, item.display_name
-                ).inc()
-            self.metrics.item_count.labels(item.item_id,
-                                           item.display_name
-                                           ).set(item.items_available)
-        except Exception:
-            self.amounts[item.item_id] = item.items_available
-        finally:
-            if self.amounts[item.item_id] != item.items_available:
-                log.info("%s - new amount: %s",
-                         item.display_name, item.items_available)
-                self.amounts[item.item_id] = item.items_available
+        if self.amounts.get(item.item_id) == item.items_available:
+            return
+        if item.item_id in self.amounts:
+            log.info("%s - new amount: %s",
+                     item.display_name, item.items_available)
+        self.metrics.item_count.labels(item.item_id,
+                                       item.display_name
+                                       ).set(item.items_available)
+        if (self.amounts.get(item.item_id) == 0 and
+                item.items_available > self.amounts.get(item.item_id)):
+            self._send_messages(item)
+            self.metrics.send_notifications.labels(
+                item.item_id, item.display_name
+            ).inc()
+        self.amounts[item.item_id] = item.items_available
 
     def _send_messages(self, item: Item) -> None:
         """

tests/test_config.py

@@ -3,7 +3,7 @@
 
 import pytest
 
-from models import Config
+from models import Config, Cron
 from models.config import DEFAULT_CONFIG
 
 
@@ -62,3 +62,52 @@ def test_token_path(temp_path: Path, monkeypatch: pytest.MonkeyPatch):
     assert config.tgtg.get("refresh_token") == "test_refresh_token"
     assert config.tgtg.get("user_id") == "test_user_id"
     assert config.tgtg.get("datadome") == "test_cookie"
+
+
+def test_ini_get(temp_path: Path):
+    config_path = Path(temp_path, "config.ini")
+
+    with open(config_path, 'w', encoding='utf-8') as file:
+        file.writelines([
+            "[MAIN]\n",
+            "Debug = true\n",
+            "ItemIDs = 23423, 32432, 234532\n",
+            "[WEBHOOK]\n",
+            "timeout = 42\n",
+            'headers = {"Accept": "json"}\n',
+            "cron = * * 1-5 * *\n",
+            'body = {"content": "${{items_available}} panier(s) à '
+            '${{price}} € \\nÀ récupérer"}'
+        ])
+
+    config = Config(config_path.absolute())
+
+    assert config.debug is True
+    assert config.item_ids == ["23423", "32432", "234532"]
+    assert config.webhook.get("timeout") == 42
+    assert config.webhook.get("headers") == {"Accept": "json"}
+    assert config.webhook.get("cron") == Cron("* * 1-5 * *")
+    assert config.webhook.get("body") == ('{"content": "${{items_available}} '
+                                          'panier(s) à ${{price}} € \n'
+                                          'À récupérer"}')
+
+
+def test_env_get(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setenv("DEBUG", "true")
+    monkeypatch.setenv("ITEM_IDS", "23423, 32432, 234532")
+    monkeypatch.setenv("WEBHOOK_TIMEOUT", "42")
+    monkeypatch.setenv("WEBHOOK_HEADERS", '{"Accept": "json"}')
+    monkeypatch.setenv("WEBHOOK_CRON", "* * 1-5 * *")
+    monkeypatch.setenv("WEBHOOK_BODY", '{"content": "${{items_available}} '
+                       'panier(s) à ${{price}} € \\nÀ récupérer"}')
+
+    config = Config()
+
+    assert config.debug is True
+    assert config.item_ids == ["23423", "32432", "234532"]
+    assert config.webhook.get("timeout") == 42
+    assert config.webhook.get("headers") == {"Accept": "json"}
+    assert config.webhook.get("cron") == Cron("* * 1-5 * *")
+    assert config.webhook.get("body") == ('{"content": "${{items_available}} '
+                                          'panier(s) à ${{price}} € \n'
+                                          'À récupérer"}')

tests/test_metrics.py

@@ -0,0 +1,11 @@
+import requests
+
+from models.metrics import Metrics
+
+
+def test_metrics():
+    metrics = Metrics(8000)
+    metrics.enable_metrics()
+    res = requests.get("http://localhost:8000")
+
+    assert res.ok