Merge pull request #39 from kamil-certat/packaging_fast_api

API rewritten to FastAPI

.github/workflows/code_style.yml

@@ -0,0 +1,38 @@
+# Check the code style
+#
+# SPDX-FileCopyrightText: 2022 CERT.at GmbH
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+name: "Check the code style"
+
+on:
+  push:
+    paths-ignore:
+      - ".github/**"
+  pull_request:
+    branches: [develop, maintenance]
+    paths-ignore:
+      - ".github/**"
+
+jobs:
+  pycodestyle:
+    name: Run pycodestyle
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+          cache: "pip"
+          cache-dependency-path: setup.py
+
+      - name: Install pycodestyle
+        run: pip install pycodestyle
+
+      - name: Run pycodestyle
+        run: |
+          pycodestyle intelmq_api tests setup.py

.github/workflows/debian-package.yml

@@ -12,7 +12,7 @@ jobs:
     name: A build on different Debian distributions
     strategy:
       matrix:
-        codename: ['buster', 'bullseye']
+        codename: ['bullseye']  # Only bullseye contains all dependencies
 
     steps:
     - name: Checkout repository

.github/workflows/mypy.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.8]
 
     steps:
     - name: Checkout repository
@@ -30,6 +30,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        pip install .
         pip install mypy
     - name: Static type checking with mypy
       run: |

.github/workflows/python-unittests.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.7, 3.8, 3.9, "3.10", 3.11]
 
     steps:
     - name: Checkout repository
@@ -27,6 +27,9 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        pip install .[dev]
     - name: Run unittests
       run: |
         python -m unittest discover -s tests/

.gitignore

@@ -6,3 +6,5 @@ build
 dist
 *.egg-info
 *.sqlite
+
+.vscode/

CHANGELOG.rst

@@ -1,10 +1,42 @@
-.. 
+..
    SPDX-FileCopyrightText: 2020-2022 Birger Schacht, Sebastian Wagner
    SPDX-License-Identifier: AGPL-3.0-or-later
 
 CHANGELOG
 =========
 
+3.1.0 (unreleased)
+----------------------
+
+**Removed:**
+
+- Removed support for Python 3.6.
+- Removed duplication of the CLI commands. Adding users could be done now only by using `scripts/intelmq-api-adduser`
+
+**Added:**
+
+- Added code style checks to CI.
+- Added interactive API documentation available at `/docs` endpoint.
+- Added the line length limit in Python files as 100 chars.
+- Added the `intelmq-api-setup-systemd` script to help with initial setup.
+
+**Changed:**
+
+- The base API framework was changed from hug to the FastAPI. All endpoints were rewritten to use it
+  and the structure of the files were changed to better align with the FastAPI concepts.
+- All endpoints returns now the proper Content-Type header.
+- On authentication error, in the returned JSON dictionary, the key with explanation was changed
+  from `errors` to `error`. This is consistent with all other error messages.
+- The startup point of the application is now `intelmq_api.main:app`
+- The API is no longer a WSGI app. The Debian package is updated to run Gunicorn as a server and
+  configure Apache2 as a proxy. If you had non-default configuration, please review new examples.
+
+**Known issues:**
+
+- The interactive API documentation on `/docs` doesn't respect setting `Authorization` header.
+- The OS native package is prepared for Debian 11 (and distribution based on it) only. For other
+  platforms, installation using `pip` is recommended.
+
 
 3.1.0 RC (2022-08-02)
 ---------------------

README.rst

@@ -1,30 +1,63 @@
-.. 
+..
    SPDX-FileCopyrightText: 2020 Birger Schacht
    SPDX-License-Identifier: AGPL-3.0-or-later
 
 ###########
 intelmq-api
 ###########
 
-|Build Status|
+|Tests Status| |Package Status|
 
-.. |Build Status| image:: https://travis-ci.com/certtools/intelmq-api.svg?branch=develop
-   :target: https://travis-ci.com/certtools/intelmq-api
+.. |Tests Status| image:: https://github.com/certtools/intelmq-api/actions/workflows/python-unittests.yml/badge.svg
+   :target: https://github.com/certtools/intelmq-api/actions/workflows/python-unittests.yml
 
-intelmq-api is a `hug <http://hug.rest>`_ based API for the `intelmq <https://github.com/certtools/intelmq/>`_ project.
+.. |Package Status| image:: https://github.com/certtools/intelmq-api/actions/workflows/debian-package.yml/badge.svg
+   :target: https://github.com/certtools/intelmq-api/actions/workflows/debian-package.yml
+
+intelmq-api is a `FastAPI <https://fastapi.tiangolo.com/>`_ based API for the `intelmq <https://github.com/certtools/intelmq/>`_ project.
 
 
 Extensive documentation regarding the installation, configuration and usage of the `intelmq-api` can be found in the `intelmq documentation <https://intelmq.readthedocs.io/en/maintenance/user/intelmq-api.html>`_.
 
+*****************
+Development usage
+*****************
+
+You could create a preferred virtual environment, and then install the package using:
+
+.. code-block:: bash
+
+   pip install -e .
+
+For development purposes, you can run the API using the `scripts/run_dev.sh` script. It serves the
+API on the local `8000` port and watches for file changes, with the automated reloading on change.
+
+The interactive documentation is served on the `/docs` endpoint.
+
+To set the API configuration, please export the `INTELMQ_API_CONFIG` environment variable with path
+to the JSON config file in the shell you want to use. For the config reference, please check the
+`intelmq_api/config.py` and the example from `contrib/api-config.json`.
+
+If you configured the session store, you will need to authorize every request. The `/v1/login`
+returns the authorization token. You can use the following command to register the user:
+
+.. code-block:: bash
+
+   ./scripts/intelmq-api-adduser --user UserName
+
+*************
+Security note
+*************
+
+Please be careful when deploying the API. At the current stage, it is not designed to run on
+publicly exposed endpoints without additional pre-cautions.
 
 *************
 Type checking
 *************
 
-Except for the parts that directly deal with ``hug``, the code can be
-typechecked with ``mypy``. To run the type checker, start with the module
-``serve``:
+The code can be typechecked with ``mypy``. To run the type checker, use:
 
 .. code-block:: bash
 
-   mypy intelmq_manager/serve.py
+   mypy intelmq_api/

contrib/api-apache.conf

@@ -1,16 +1,9 @@
-# SPDX-FileCopyrightText: 2020 Birger Schacht
-#
+# SPDX-FileCopyrightText: 2022 CERT.at GmbH <https://cert.at/>
 # SPDX-License-Identifier: CC0-1.0
-
-#Override the default configuration file path using the
-#INTELMQ_API_CONFIG environment variable
-#SetEnv INTELMQ_API_CONFIG /etc/intelmq/api-config.json
-<IfModule mod_wsgi.c>
-WSGIApplicationGroup %{GLOBAL}
-WSGIPassAuthorization On
-WSGIScriptAlias /intelmq /usr/lib/python3/dist-packages/intelmq_api/intelmq-api.wsgi
-</IfModule>
 
-<Directory /usr/lib/python3/dist-packages/intelmq_api/>
-   Require all granted
-</Directory>
+
+# If you want to change default location, please align the ROOT_PATH in the service configuration
+<Location /intelmq/>
+	ProxyPass unix:/usr/lib/python3/dist-packages/intelmq_api/intelmq_api.sock|http://127.0.0.1/
+	ProxyPassReverse unix:/usr/lib/python3/dist-packages/intelmq_api/intelmq_api.sock|http://127.0.0.1/
+</Location>

contrib/intelmq-api.service

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: 2022 CERT.at GmbH <https://cert.at/>
+# SPDX-License-Identifier: CC0-1.0
+
+[Unit]
+Description=Gunicorn deamon to serve the IntelMQ API
+Requires=intelmq-api.socket
+After=network.target
+
+[Service]
+
+# To override settings path, use e.g.:
+# Environment="INTELMQ_API_CONFIG=/etc/intelmq/api-config.json"
+
+Environment="ROOT_PATH=/intelmq"
+User=www-data
+Group=www-data
+RuntimeDirectory=gunicorn
+WorkingDirectory=/usr/lib/python3/dist-packages/intelmq_api/
+ExecStart=/usr/bin/gunicorn intelmq_api.main:app --workers 4 --worker-class uvicorn.workers.UvicornWorker --bind unix:intelmq_api.sock
+ExecReload=/bin/kill -s HUP $MAINPID
+KillMode=mixed
+TimeoutStopSec=5
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

contrib/intelmq-api.socket

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2022 CERT.at GmbH <https://cert.at/>
+# SPDX-License-Identifier: CC0-1.0
+
+[Unit]
+Description=The socket to handle IntelMQ API requests
+
+[Socket]
+ListenStream=/usr/lib/python3/dist-packages/intelmq_api/intelmq_api.sock
+SocketUser=www-data
+
+[Install]
+WantedBy=sockets.target

debian/changelog

@@ -1,3 +1,11 @@
+intelmq-api (3.1.0~rc2-1) UNRELEASED; urgency=medium
+
+  * API rewritten to use the FastAPI framework
+  * Package updated to setup gunicorn with Apache2, without WSGI module
+  * Server configuration is changed.
+
+ -- Kamil Mankowski <[email protected]>  Tue, 20 Dec 2022 16:23:27 +0100
+
 intelmq-api (3.1.0~rc-1) stable; urgency=medium
 
   * No changes. Just keeping version numbers in sync.

debian/compat

@@ -1 +1 @@
-9
+10

debian/control

@@ -2,12 +2,15 @@ Source: intelmq-api
 Section: python
 Priority: optional
 Maintainer: IntelMQ Team <[email protected]>
-Uploaders: Birger Schacht <schacht@cert.at>
-Build-Depends: debhelper (>= 9),
+Uploaders: Sebastian Wagner <sebix@sebix.at>
+Build-Depends: debhelper (>= 10),
                dh-python,
                dh-exec,
                python3-all,
-               python3-setuptools
+               python3-setuptools,
+               python3-fastapi,
+               python3-typing-extensions,
+               python3-multipart
 Standards-Version: 4.5.0
 Homepage: https://github.com/certtools/intelmq-api
 Rules-Requires-Root: no
@@ -17,15 +20,20 @@ Architecture: all
 Depends: ${misc:Depends},
          ${python3:Depends},
          ${shlibs:Depends},
-         python3-requests,
-         python3-hug,
+         python3-fastapi,
+         python3-typing-extensions,
+         python3-multipart,
+         python3-uvicorn,
+         python3-uvloop,
+         python3-httptools,
+         gunicorn,
          intelmq,
          sudo,
          dbconfig-sqlite3 | dbconfig-no-thanks
 Breaks: intelmq-manager (< 2.3)
 Replaces: intelmq-manager (< 2.3)
-Recommends: libapache2-mod-wsgi-py3, apache2
-Description: HUG based API for the intelmq project
+Recommends: apache2
+Description: FastAPI-based API for the intelmq project
  IntelMQ-API provides a simple API for controlling and configuring an IntelMQ
  installation. IntelMQ is a solution for IT security teams for collecting and
  processing security feeds.

debian/install

@@ -1,3 +1,5 @@
 #!/usr/bin/dh-exec
 contrib/api-sudoers.conf => /etc/sudoers.d/01_intelmq-api
 contrib/api-session.sql => /usr/share/dbconfig-common/data/intelmq-api/install/sqlite3
+contrib/intelmq-api.service => /lib/systemd/system/intelmq-api.service
+contrib/intelmq-api.socket => /lib/systemd/system/intelmq-api.socket

debian/postinst

@@ -3,11 +3,13 @@
 if [ "$1" = "configure" ] ; then
     if [ -e /usr/share/apache2/apache2-maintscript-helper ] ; then
         . /usr/share/apache2/apache2-maintscript-helper
+		apache2_invoke enmod proxy_http || true
         apache2_invoke enconf intelmq-api.conf || true
 
         #adduser www-data intelmq
-	usermod -a -G intelmq www-data
+		usermod -a -G intelmq www-data
     fi
+
 fi
 
 # Source debconf library.

debian/rules

@@ -9,3 +9,7 @@ override_dh_auto_build:
 override_dh_auto_install:
 	dh_auto_install
 	sed -i "s#/etc/intelmq/api-session.sqlite#/var/lib/dbconfig-common/sqlite3/intelmq-api/intelmqapi#" debian/intelmq-api/etc/intelmq/api-config.json
+	# Remove files placed by setuptools, not needed with the DEB package
+	rm debian/intelmq-api/etc/intelmq/intelmq-api.service
+	rm debian/intelmq-api/etc/intelmq/intelmq-api.socket
+	rm debian/intelmq-api/etc/intelmq/api-sudoers.conf