Initial commit

.dockerignore

@@ -0,0 +1,4 @@
+__pycache__
+promdata
+grafdata
+feedsmon.yml

.gitignore

@@ -0,0 +1,4 @@
+__pycache__
+promdata
+grafdata
+feedsmon.yml

Dockerfile

@@ -0,0 +1,29 @@
+# Use an official Python runtime as the base image
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y \
+  build-essential \
+  flex \
+  bison \
+  libtool \
+  make \
+  automake \
+  autoconf \
+  build-essential \
+  && rm -rf /var/lib/apt/lists/*
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Copy the requirements.txt file and install dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the application's source code
+COPY . .
+
+# Set any environment variables, if needed
+EXPOSE 8000
+
+# Specify the command to run your application
+CMD [ "python", "main.py" ]

cfg.py

@@ -0,0 +1,11 @@
+from yaml import load
+
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
+
+
+def load_config():
+    with open("feedsmon.yml", "r") as f:
+        return load(f, Loader=Loader)

data_feed_metrics.py

@@ -0,0 +1,18 @@
+from prometheus_client import Counter, Gauge, Info, start_http_server
+
+metrics = {
+    "data_feed": Gauge("data_feed", "Numeric data feed", ["project", "code"]),
+    "data_feed_last_error": Info(
+        "data_feed_last_error", "Most recent error detected", ["project", "code"]
+    ),
+    "data_feed_error_count": Counter(
+        "data_feed_error_count",
+        "Number of errors whilst retrieving data feed",
+        ["project", "code"],
+    ),
+    "data_feed_update_count": Counter(
+        "data_feed_update_count",
+        "Number of times data feed retrieved",
+        ["project", "code"],
+    ),
+}

docker-compose.yml

@@ -0,0 +1,31 @@
+version: '3'
+services:
+  prometheus:
+    image: prom/prometheus
+    ports:
+      - 9090:9090
+    user: "1000"
+    volumes:
+      - ./promdata:/etc/prometheus
+      - ./promdata/data:/prometheus/data
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    restart: always
+
+  grafana:
+    image: grafana/grafana
+    ports:
+      - 3000:3000
+    user: "1000"
+    volumes:
+      - ./grafdata:/var/lib/grafana
+    depends_on:
+      - prometheus
+    restart: always
+
+  feedsmon:
+    build:
+      context: .
+    ports:
+      - 8000:8000
+    restart: always

main.py

@@ -0,0 +1,79 @@
+import datetime
+import time
+
+import pyjq
+import requests
+
+from cfg import load_config
+
+
+def init_metrics(data_feed_sources):
+    for data_feed_source in data_feed_sources:
+        metrics["data_feed"].labels(
+            data_feed_source["project"], data_feed_source["code"]
+        ).set(0)
+        metrics["data_feed_last_error"].labels(
+            data_feed_source["project"], data_feed_source["code"]
+        ).info({"status": "", "timestamp": "", "error": ""})
+        print(
+            f"Initialised metrics for {data_feed_source['project']}-{data_feed_source['code']}",
+        )
+
+
+def process_request(data_feed_sources, t):
+    for data_feed_source in data_feed_sources:
+        try:
+            response = requests.get(
+                data_feed_source["url"], headers=data_feed_source["headers"]
+            )
+            if response.status_code != 200:
+                current_datetime = datetime.datetime.now()
+                metrics["data_feed_last_error"].labels(
+                    data_feed_source["project"], data_feed_source["code"]
+                ).info(
+                    {
+                        "status": response.status_code,
+                        "timestamp": current_datetime.strftime("%Y-%m-%d %H:%M:%S"),
+                        "error": response.text,
+                    }
+                )
+                metrics["data_feed_error_count"].labels(
+                    data_feed_source["project"], data_feed_source["code"]
+                ).inc()
+            else:
+                data = response.json()
+
+                price_str = pyjq.first(data_feed_source["jq"], data)
+                price = float(price_str) / 10 ** data_feed_source["decimals"]
+                metrics["data_feed"].labels(
+                    data_feed_source["project"], data_feed_source["code"]
+                ).set(price)
+                metrics["data_feed_update_count"].labels(
+                    data_feed_source["project"], data_feed_source["code"]
+                ).inc()
+            print(f"{data_feed_source['code']} price: {price}")
+
+        except Exception as e:
+            current_datetime = datetime.datetime.now()
+            metrics["data_feed_last_error"].labels(
+                data_feed_source["project"], data_feed_source["code"]
+            ).info(
+                {
+                    "status": "0",
+                    "timestamp": current_datetime.strftime("%Y-%m-%d %H:%M:%S"),
+                    "error": f"{e.args[0]}",
+                }
+            )
+            metrics["data_feed_error_count"].labels(
+                data_feed_source["project"], data_feed_source["code"]
+            ).inc()
+
+    time.sleep(t)
+
+
+if __name__ == "__main__":
+    start_http_server(8000)
+    data_feed_sources = load_config()
+    init_metrics(data_feed_sources)
+    while True:
+        process_request(data_feed_sources, 60)

requirements.txt

@@ -0,0 +1,5 @@
+prometheus-client
+requests
+pyjq
+inflection
+pyyaml