Update package

.coveragerc

@@ -0,0 +1,23 @@
+[run]
+branch = True
+source =
+    dump1090exporter
+
+[paths]
+source =
+    src/dump1090exporter
+    **/site-packages/dump1090exporter
+
+[report]
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    if self.debug
+    raise AssertionError
+    raise NotImplementedError
+    if 0:
+    if __name__ == .__main__.:
+ignore_errors = True
+
+[html]
+directory = coverage

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: Python Package Workflow
+name: CI Pipeline
 
 on:
   push:
@@ -34,18 +34,25 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
           ${{ runner.os }}-
-    - name: Check style
+    - name: Check code style
       run: |
         make check-style
+    - name: Install Package
+      run: |
+        pip install .
     - name: Check lint
       run: |
         make check-lint
     - name: Check types
       run: |
         make check-types
-    - name: Check installation
+    - name: Check tests
       run: |
-        pip install .
+        make test
+    - name: Check code coverage
+      if: ${{ matrix.python-version == '3.9' }}
+      run: |
+        make coverage
     - name: Generate package
       run: |
         make dist

Makefile

@@ -41,18 +41,42 @@ scrub:
 	git clean -x -f -d
 
 
+# help: test                  - run tests
+.PHONY: test
+test:
+	@python -m unittest discover -s tests
+
+
+# help: test-verbose          - run tests [verbosely]
+.PHONY: test-verbose
+test-verbose:
+	@python -m unittest discover -s tests -v
+
+
+# help: coverage              - perform test coverage checks
+.PHONY: coverage
+coverage:
+	@coverage erase
+	@rm -f .coverage.unit
+	@COVERAGE_FILE=.coverage.unit coverage run -m unittest discover -s tests -v
+	@coverage combine
+	@coverage report
+	@coverage html
+	@coverage xml
+
+
 # help: check-style           - perform code format compliance check
 .PHONY: check-style
 check-style:
 	@isort . --check-only --profile black
-	@black --check src/dump1090exporter setup.py
+	@black --check src/dump1090exporter setup.py tests
 
 
 # help: style                 - perform code style formatting
 .PHONY: style
 style:
 	@isort . --profile black
-	@black src/dump1090exporter setup.py
+	@black src/dump1090exporter setup.py tests
 
 
 # help: check-types           - check type hint annotations
@@ -64,7 +88,7 @@ check-types:
 # help: check-lint            - run static analysis checks
 .PHONY: check-lint
 check-lint:
-	@pylint --rcfile=.pylintrc dump1090exporter setup.py
+	@pylint --rcfile=.pylintrc dump1090exporter setup.py tests
 
 
 # help: dist                  - create a wheel distribution package

requirements.dev.txt

@@ -1,5 +1,7 @@
-black
-isort
-mypy
-pylint
-wheel
+asynctest==0.13.0
+black==21.9b0
+coverage==6.0
+isort==5.9.3
+mypy==0.910
+pylint==2.11.1
+wheel==0.37.0

requirements.txt

@@ -1,2 +1,2 @@
-aiohttp
-aioprometheus[aiohttp]==21.8.0
+aiohttp==3.7.4.post0
+aioprometheus[aiohttp]==21.9.0

setup.py

@@ -49,7 +49,7 @@ def parse_requirements(filename):
         install_requires=parse_requirements("requirements.txt"),
         extras_require={
             "develop": parse_requirements("requirements.dev.txt"),
-            "uvloop": ["uvloop"],
+            "uvloop": ["uvloop==0.16.0"],
         },
         classifiers=[
             "Development Status :: 5 - Production/Stable",

src/dump1090exporter/__init__.py

@@ -1,3 +1,3 @@
 from .exporter import Dump1090Exporter
 
-__version__ = "21.9.0"
+__version__ = "21.10.0"

src/dump1090exporter/exporter.py

@@ -9,12 +9,12 @@
 import json
 import logging
 import math
-from asyncio.events import AbstractEventLoop
 from math import asin, atan, cos, degrees, radians, sin, sqrt
 from typing import Any, Dict, Sequence, Tuple, Union
 
 import aiohttp
-from aioprometheus import Gauge, Service
+from aioprometheus import Gauge
+from aioprometheus.service import Service
 
 from .metrics import Specs
 
@@ -166,6 +166,22 @@ def haversine_distance(
     return distance
 
 
+def create_gauge_metric(label: str, doc: str, prefix: str = "") -> Gauge:
+    """Create a Gauge metric
+
+    :param label: A label for the metric.
+
+    :param doc: A help string for the metric.
+
+    :param prefix: An optional prefix for the metric label that applies a
+        common start string to the metric which simplifies locating the
+        metrics in Prometheus because they will all be grouped together when
+        searching.
+    """
+    gauge = Gauge(f"{prefix}{label}", doc)
+    return gauge
+
+
 class Dump1090Exporter:
     """
     This class is responsible for fetching, parsing and exporting dump1090
@@ -175,7 +191,7 @@ class Dump1090Exporter:
     def __init__(
         self,
         resource_path: str,
-        host: str = None,
+        host: str = "",
         port: int = 9105,
         aircraft_interval: int = 10,
         stats_interval: int = 60,
@@ -184,7 +200,6 @@ def __init__(
         time_periods: Sequence[str] = ("last1min",),
         origin: PositionType = None,
         fetch_timeout: float = 2.0,
-        loop: AbstractEventLoop = None,
     ) -> None:
         """
         :param resource_path: The base dump1090 resource address. This can be
@@ -214,12 +229,12 @@ def __init__(
           be zero.
         :param fetch_timeout: The number of seconds to wait for a response
           from dump1090.
-        :param loop: the event loop.
         """
         self.resources = build_resources(resource_path)
-        self.loop = loop or asyncio.get_event_loop()
+        self.loop = asyncio.get_event_loop()
         self.host = host
         self.port = port
+        self.prefix = "dump1090_"
         self.receiver_interval = datetime.timedelta(seconds=receiver_interval)
         self.receiver_interval_origin_ok = datetime.timedelta(
             seconds=receiver_interval_origin_ok
@@ -298,18 +313,13 @@ def initialise_metrics(self) -> None:
         # aircraft
         d = self.metrics["aircraft"]
         for (name, label, doc) in Specs["aircraft"]:  # type: ignore
-            d[name] = self._create_gauge_metric(label, doc)
+            d[name] = create_gauge_metric(label, doc, prefix=self.prefix)
 
         # statistics
         for group, metrics_specs in Specs["stats"].items():  # type: ignore
             d = self.metrics["stats"].setdefault(group, {})
             for name, label, doc in metrics_specs:
-                d[name] = self._create_gauge_metric(label, doc)
-
-    def _create_gauge_metric(self, label, doc):
-        gauge = Gauge(f"dump1090_{label}", doc)
-        self.svr.register(gauge)
-        return gauge
+                d[name] = create_gauge_metric(label, doc, prefix=self.prefix)
 
     async def _fetch(
         self,

src/dump1090exporter/metrics.py

@@ -31,7 +31,7 @@
     dump1090_stats_messages_total
 
 There are multiple sections in the dump1090 stats data file. The Prometheus
-multi-dimensional metrics label are used to expose these. So to obtain the
+multi-dimensional metrics labels are used to expose these. So, to obtain the
 stats metrics for the last1min group you would use a metrics label of:
 
 .. code-block:: console

tests/golden-data/aircraft.json

@@ -0,0 +1,7 @@
+{ "now" : 1633214709.0,
+  "messages" : 318553,
+  "aircraft" : [
+    {"hex":"75025b","version":2,"sil_type":"perhour","mlat":[],"tisb":[],"messages":25,"seen":102.4,"rssi":-1.7},
+    {"hex":"7c495b","flight":"RXA4362 ","alt_baro":7650,"alt_geom":7300,"gs":156.0,"track":269.6,"geom_rate":1024,"squawk":"4023","category":"A2","lat":-34.909653,"lon":138.264509,"nic":8,"rc":186,"seen_pos":2.4,"version":1,"nic_baro":1,"nac_p":9,"nac_v":2,"sil":2,"sil_type":"unknown","mlat":[],"tisb":[],"messages":303,"seen":2.4,"rssi":-2.1}
+  ]
+}

tests/golden-data/receiver.json

@@ -0,0 +1 @@
+{ "version" : "6.1", "refresh" : 1000, "history" : 120, "lat" : -34.928500, "lon" : 138.600700 }

tests/golden-data/stats.json

@@ -0,0 +1,7 @@
+{
+"latest":{"start":1633214698.9,"end":1633214698.9,"local":{"samples_processed":0,"samples_dropped":0,"modeac":0,"modes":0,"bad":0,"unknown_icao":0,"accepted":[0,0],"strong_signals":0},"remote":{"modeac":0,"modes":0,"bad":0,"unknown_icao":0,"accepted":[0,0]},"cpr":{"surface":0,"airborne":0,"global_ok":0,"global_bad":0,"global_range":0,"global_speed":0,"global_skipped":0,"local_ok":0,"local_aircraft_relative":0,"local_receiver_relative":0,"local_skipped":0,"local_range":0,"local_speed":0,"filtered":0},"altitude_suppressed":0,"cpu":{"demod":0,"reader":0,"background":0},"tracks":{"all":0,"single_message":0,"unreliable":0},"messages":0,"messages_by_df":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},
+"last1min":{"start":1633214638.8,"end":1633214698.9,"local":{"samples_processed":144048128,"samples_dropped":0,"modeac":0,"modes":1627196,"bad":3818271,"unknown_icao":664144,"accepted":[48,45],"signal":-2.5,"noise":-9.3,"peak_signal":-1.2,"strong_signals":80},"remote":{"modeac":0,"modes":0,"bad":0,"unknown_icao":0,"accepted":[0,0]},"cpr":{"surface":0,"airborne":28,"global_ok":17,"global_bad":0,"global_range":0,"global_speed":0,"global_skipped":0,"local_ok":11,"local_aircraft_relative":0,"local_receiver_relative":0,"local_skipped":0,"local_range":0,"local_speed":0,"filtered":0},"altitude_suppressed":0,"cpu":{"demod":12825,"reader":3743,"background":471},"tracks":{"all":4,"single_message":5,"unreliable":5},"messages":93,"messages_by_df":[0,0,0,0,0,0,0,0,0,0,0,32,0,0,0,0,0,57,1,0,1,2,0,0,0,0,0,0,0,0,0,0]},
+"last5min":{"start":1633214398.8,"end":1633214698.9,"local":{"samples_processed":720109568,"samples_dropped":0,"modeac":0,"modes":8129321,"bad":19068879,"unknown_icao":3319869,"accepted":[177,164],"signal":-2.3,"noise":-9.3,"peak_signal":-0.9,"strong_signals":309},"remote":{"modeac":0,"modes":0,"bad":0,"unknown_icao":0,"accepted":[0,0]},"cpr":{"surface":0,"airborne":87,"global_ok":55,"global_bad":0,"global_range":0,"global_speed":0,"global_skipped":0,"local_ok":25,"local_aircraft_relative":0,"local_receiver_relative":0,"local_skipped":7,"local_range":0,"local_speed":0,"filtered":0},"altitude_suppressed":0,"cpu":{"demod":64327,"reader":18583,"background":2320},"tracks":{"all":20,"single_message":23,"unreliable":23},"messages":341,"messages_by_df":[1,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,2,218,6,0,5,9,0,0,0,0,0,0,0,0,0,0]},
+"last15min":{"start":1633213798.9,"end":1633214698.9,"local":{"samples_processed":2160066560,"samples_dropped":0,"modeac":0,"modes":24403357,"bad":57271795,"unknown_icao":9959385,"accepted":[327,328],"signal":-2.7,"noise":-9.3,"peak_signal":-0.9,"strong_signals":516},"remote":{"modeac":0,"modes":0,"bad":0,"unknown_icao":0,"accepted":[0,0]},"cpr":{"surface":0,"airborne":150,"global_ok":99,"global_bad":0,"global_range":0,"global_speed":0,"global_skipped":0,"local_ok":41,"local_aircraft_relative":0,"local_receiver_relative":0,"local_skipped":10,"local_range":0,"local_speed":0,"filtered":0},"altitude_suppressed":0,"cpu":{"demod":192831,"reader":55958,"background":6841},"tracks":{"all":61,"single_message":60,"unreliable":60},"messages":655,"messages_by_df":[2,0,0,0,0,0,0,0,0,0,0,210,0,0,0,0,2,373,30,0,22,16,0,0,0,0,0,0,0,0,0,0]},
+"total":{"start":1633116898.7,"end":1633214698.9,"local":{"samples_processed":234730422272,"samples_dropped":0,"modeac":0,"modes":2651922156,"bad":1927197336,"unknown_icao":1082300238,"accepted":[201103,112954],"signal":-2.4,"noise":-9.6,"peak_signal":-0.7,"strong_signals":263179},"remote":{"modeac":0,"modes":4491,"bad":0,"unknown_icao":0,"accepted":[4491,0]},"cpr":{"surface":4,"airborne":81431,"global_ok":73315,"global_bad":0,"global_range":0,"global_speed":0,"global_skipped":32,"local_ok":6958,"local_aircraft_relative":0,"local_receiver_relative":0,"local_skipped":1162,"local_range":0,"local_speed":0,"filtered":0},"altitude_suppressed":0,"cpu":{"demod":20955239,"reader":6090253,"background":768489},"tracks":{"all":6277,"single_message":5999,"unreliable":6021},"messages":318548,"messages_by_df":[9039,0,0,0,2720,3530,0,0,0,0,0,69845,0,0,0,0,277,197439,7698,0,15406,12594,0,0,0,0,0,0,0,0,0,0]}
+}

tests/test_exporter.py

@@ -0,0 +1,100 @@
+import asyncio
+from pathlib import Path
+from typing import Optional
+
+import asynctest
+from aiohttp import ClientSession, web
+from aioprometheus import REGISTRY
+
+import dump1090exporter.exporter
+import dump1090exporter.metrics
+from dump1090exporter import Dump1090Exporter
+
+GOLDEN_DATA_DIR = Path(__file__).parent / "golden-data"
+AIRCRAFT_DATA_FILE = GOLDEN_DATA_DIR / "aircraft.json"
+STATS_DATA_FILE = GOLDEN_DATA_DIR / "stats.json"
+RECEIVER_DATA_FILE = GOLDEN_DATA_DIR / "receiver.json"
+TEST_ORIGIN = (-34.928500, 138.600700)  # (lat, lon)
+
+
+class Dump1090ServiceEmulator:
+    """This class implements a HTTP server that emulates the dump1090 service"""
+
+    def __init__(self):  # pylint: disable=missing-function-docstring
+        self._runner = None  # type: Optional[web.AppRunner]
+        self.url = None  # type: Optional[str]
+        self.paths = {
+            "/aircraft.json": AIRCRAFT_DATA_FILE,
+            "/stats.json": STATS_DATA_FILE,
+            "/receiver.json": RECEIVER_DATA_FILE,
+        }
+
+    async def handle_request(self, request):
+        """Handle a HTTP request for a dump1090 resource"""
+        if request.path not in self.paths:
+            raise Exception(f"Unhandled path: {request.path}")
+
+        data_file = self.paths[request.path]
+        with data_file.open("rt") as f:
+            content = f.read()
+        return web.Response(status=200, body=content, content_type="application/json")
+
+    async def start(self, addr="127.0.0.1", port=None):
+        """Start the dump1090 service emulator"""
+        app = web.Application()
+        app.add_routes(
+            [web.get(request_path, self.handle_request) for request_path in self.paths]
+        )
+        self._runner = web.AppRunner(app)
+        await self._runner.setup()
+        site = web.TCPSite(self._runner, addr, port)
+        await site.start()
+        self.url = site.name
+
+    async def stop(self):
+        """Stop the dump1090 service emulator"""
+        await self._runner.cleanup()
+
+
+class TestExporter(asynctest.TestCase):  # pylint: disable=missing-class-docstring
+    def tearDown(self):
+        REGISTRY.clear()
+
+    async def test_exporter(self):
+        """Check dump1090exporter application"""
+        # Start a fake dump1090 service that the exporter can scrape
+        ds = Dump1090ServiceEmulator()
+        try:
+            await ds.start()
+
+            # Start the dump1090exporter
+            de = Dump1090Exporter(
+                resource_path=ds.url,
+                origin=TEST_ORIGIN,
+            )
+
+            await de.start()
+            await asyncio.sleep(0.3)
+
+            # Scrape the dump1090exporter just as Prometheus would
+            async with ClientSession() as session:
+                async with session.get(de.svr.metrics_url, timeout=0.3) as resp:
+                    if not resp.status == 200:
+                        raise Exception(f"Fetch failed {resp.status}: {resp.url()}")
+                    data = await resp.text()
+
+            # Check that expected metrics are present in the response
+            specs = dump1090exporter.metrics.Specs
+            for _attr, label, _doc in specs["aircraft"]:
+                self.assertIn(f"{de.prefix}{label}{{", data)
+            for _group_name, group_metrics in specs["stats"].items():
+                for _attr, label, _doc in group_metrics:
+                    self.assertIn(f"{de.prefix}{label}{{", data)
+
+            await de.stop()
+
+            # check calling stop again does not raise errors
+            await de.stop()
+
+        finally:
+            await ds.stop()

tests/test_metrics.py

@@ -0,0 +1,27 @@
+import unittest
+
+import dump1090exporter.metrics
+
+
+class TestMetrics(unittest.TestCase):
+    """Check metrics spec structure"""
+
+    def test_specification(self):
+        """check structure of specification"""
+        self.assertIsInstance(dump1090exporter.metrics.Specs, dict)
+
+        self.assertIn("aircraft", dump1090exporter.metrics.Specs)
+        v = dump1090exporter.metrics.Specs["aircraft"]
+        self.assertIsInstance(v, tuple)
+        for i in v:
+            self.assertIsInstance(i, tuple)
+            self.assertEqual(len(i), 3)
+
+        self.assertIn("stats", dump1090exporter.metrics.Specs)
+        v = dump1090exporter.metrics.Specs["stats"]
+        self.assertIsInstance(v, dict)
+        for k1, v1 in v.items():
+            self.assertIsInstance(k1, str)
+            for i in v1:
+                self.assertIsInstance(i, tuple)
+                self.assertEqual(len(i), 3)