diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0b6a3557a..da50930c0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.25.7 +current_version = 1.26.0 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-04-20T14:15:34Z +search = {current_version} 2023-04-20T20:42:31Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-04-20T14:15:34Z +values = 2023-04-20T20:42:31Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index 0ee26f17f..e0b43df42 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,64 @@ [//]: # (list changes here, using '-' for each new entry, remove this when items are added) +[1.26.0](https://github.com/bird-house/birdhouse-deploy/tree/1.26.0) (2023-04-20) +------------------------------------------------------------------------------------------------------------------ + + +## Breaking changes + +- CanarieAPI: update to `0.7.1`. + + - The Docker running `CanarieAPI` is now using Python 3 (since `0.4.x` tags). + Configurations need to be updated if any specific Python 2 definitions were used. + See [2to3](https://docs.python.org/3/library/2to3.html) to help migrate configurations automatically if necessary. + - Update the [CanarieAPI configuration](birdhouse/config/canarie-api/docker_configuration.py.template) to use + Python 3.x executable code. + +## Changes + +- CanarieAPI: update to `0.7.1`. + + - The server node now provides a generic ``server`` configuration for the current ``platform`` definition. + - Added multiple missing docuementation references for all the services included within `CanarieAPI` configurations. + - With new `CanarieAPI` version, a slightly improved UI with more service details are provided for the active server: + +![image](https://user-images.githubusercontent.com/19194484/232822454-e39c0111-54dc-4f9b-adf6-5ea6e59d67e3.png) + +- Add optional variables witht defaults to define reference Docker image version tags. + + Following optional variables are defined by default. These are used as reference in the respective Docker compose + service definition of these components, as well as in their `CanarieAPI` configuration to retrieve the release time + of the tag, and refer to relevant URL references as needed. + + - `CATALOG_VERSION` + - `FINCH_VERSION` + - `FLYINGPIGEON_VERSION` + - `GEOSERVER_VERSION` + - `HUMMINGBIRD_VERSION` + - `MALLEEFOWL_VERSION` + - `RAVEN_VERSION` + +## Fixes: + +- CanarieAPI: update to `0.7.1`. + + - Fixes an `AttributeError` raised due to misconfiguration of the Web Application with Flask 2.x definitions + (relates to [Ouranosinc/CanarieAPI#10](https://github.com/Ouranosinc/CanarieAPI/pull/10)). + - Skip over `0.4.x`, `0.5.x`, `0.6.x` versions to avoid issue related to `cron` job monitoring and log parser + command failures in order to collect configured service statistics and statuses + (see also [Ouranosinc/CanarieAPI#14](https://github.com/Ouranosinc/CanarieAPI/pull/14)). + +- Weaver: update CanarieAPI monitoring definitions + - Move monitoring of public endpoint under [optional-components/canarie-api-full-monitoring][canarie-monitor]. + - Add monitoring of private endpoint by default when using Weaver component. + +- Cowbird: update CanarieAPI monitoring definitions + - Add monitoring of public endpoint under [optional-components/canarie-api-full-monitoring][canarie-monitor]. + - Add public Magpie permission on Cowbird entrypoint only to allow its monitoring. + +[canarie-monitor]: birdhouse/optional-components/canarie-api-full-monitoring + [1.25.7](https://github.com/bird-house/birdhouse-deploy/tree/1.25.7) (2023-04-20) ------------------------------------------------------------------------------------------------------------------ diff --git a/Makefile b/Makefile index 178e0bc01..2b2d09197 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.25.7 +override APP_VERSION := 1.26.0 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index 84d9ef1c2..36fabf6a3 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.25.7.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.0.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.25.7...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.0...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.25.7-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.0-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.25.7 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.0 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index c123b0c56..9d503a8ff 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.25.7 2023-04-20T14:15:34Z +1.26.0 2023-04-20T20:42:31Z diff --git a/birdhouse/components/cowbird/.gitignore b/birdhouse/components/cowbird/.gitignore index e309900af..e553e9cbf 100644 --- a/birdhouse/components/cowbird/.gitignore +++ b/birdhouse/components/cowbird/.gitignore @@ -3,6 +3,8 @@ config/cowbird/config.yml config/cowbird/cowbird.ini config/magpie/config.yml config/proxy/conf.extra-service.d/cowbird.conf +config/proxy/canarie_api_monitoring.py # Old paths. Keep these so that old config files remain uncommittable after updates. conf.extra-service.d/cowbird.conf +config/canarie-api/cowbird_config.py diff --git a/birdhouse/components/cowbird/config/magpie/config.yml.template b/birdhouse/components/cowbird/config/magpie/config.yml.template index fef5e6484..cc9cfdd30 100644 --- a/birdhouse/components/cowbird/config/magpie/config.yml.template +++ b/birdhouse/components/cowbird/config/magpie/config.yml.template @@ -18,6 +18,17 @@ permissions: permission: write group: administrators action: create + # Allow validating/monitoring the service entrypoint and its version + # Everything else should remain admin-protected since it manages other user/group permissions. + - service: cowbird + permission: read-match + group: anonymous + action: create + - service: cowbird + resource: /version + permission: read-match + group: anonymous + action: create # Definitions of all the webhooks urls that will be called when creating or deleting a user. webhooks: diff --git a/birdhouse/components/cowbird/config/proxy/canarie_api_monitoring.py.template b/birdhouse/components/cowbird/config/proxy/canarie_api_monitoring.py.template new file mode 100644 index 000000000..f97e0d380 --- /dev/null +++ b/birdhouse/components/cowbird/config/proxy/canarie_api_monitoring.py.template @@ -0,0 +1,38 @@ +SERVICES['Cowbird'] = { + 'info': { + 'name': 'Cowbird', + 'synopsis': 'Cowbird is a middleware that manages interactions between various birds of the bird-house stack.', + 'version': "${COWBIRD_VERSION}", + 'institution': 'Ouranos, CRIM', + 'releaseTime': get_release_time_from_repo_tag("github", "Ouranosinc/cowbird", "${COWBIRD_VERSION}"), + 'researchSubject': 'Any', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Security', + 'tags': ['Security', 'Management', 'Access', 'Policy Decision Point'] + }, + 'stats': { + 'method': '.*', + 'route': "/cowbird/.*" + }, + 'redirect': { + 'doc': 'https://pavics-cowbird.readthedocs.io/', + 'releasenotes': 'https://github.com/Ouranosinc/cowbird//blob/master/CHANGES.rst', + 'support': 'https://github.com/Ouranosinc/cowbird//issues', + 'source': 'https://github.com/Ouranosinc/cowbird/', + 'tryme': 'https://${PAVICS_FQDN_PUBLIC}/cowbird/', + 'licence': 'https://github.com/Ouranosinc/cowbird//blob/${COWBIRD_VERSION}/LICENSE', + 'provenance': 'https://github.com/Ouranosinc/cowbird/' + }, + "monitoring": { + "Cowbird": { + 'request': { + 'url': 'http://cowbird:7000/' + }, + 'response': { + 'text': '\{.*"title":.*"Cowbird REST API".*\}' + } + } + } +} + +# vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/components/cowbird/config/proxy/docker-compose-extra.yml b/birdhouse/components/cowbird/config/proxy/docker-compose-extra.yml index 88b6e85ba..d876d89c2 100644 --- a/birdhouse/components/cowbird/config/proxy/docker-compose-extra.yml +++ b/birdhouse/components/cowbird/config/proxy/docker-compose-extra.yml @@ -5,5 +5,12 @@ services: proxy: volumes: - ./components/cowbird/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/cowbird:ro + # NOTE: + # Prefix '0_' to the mounted file name to ensure it is loaded first by 'birdhouse/config/canarie-api/docker_configuration.py' + # This ensures that the alphabetical loading order it defines will first load the default configs, then will load + # the Cowbird config, and then all python configuration within 'optional-components/canarie-api-full-monitoring'. + # It is important to load Cowbird before, as the full monitoring need to know if 'Cowbird' service is added to the + # stack in order to apply the corresponding public endpoint conditionally. + - ./components/cowbird/config/proxy/canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/0_cowbird_config.py:ro links: - cowbird diff --git a/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template index 397f646d0..5cce6ab9b 100644 --- a/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/weaver/config/canarie-api/canarie_api_monitoring.py.template @@ -1,9 +1,38 @@ -SERVICES['node']['monitoring'].update({ - 'Weaver': { - 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}/weaver' - }, +SERVICES['Weaver'] = { + 'info': { + 'name': 'Weaver', + 'synopsis': 'Weaver is an OGC-API flavored Execution Management Service (EMS) that allows the execution of workflows chaining various applications and Web Processing Services (WPS) inputs and outputs. Remote execution is deferred by the EMS to one or many Application, Deployment and Execution Service (ADES) or remote service providers, and employs Common Workflow Language (CWL) configurations to define an OGC Application Package deployed for each process.', + 'version': "${WEAVER_VERSION}", + 'institution': 'CRIM', + 'releaseTime': get_release_time_from_repo_tag("github", "crim-ca/weaver", "${WEAVER_VERSION}"), + 'researchSubject': 'Any', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Processing', + 'tags': ['Workflow', 'Processes', 'Deployment', 'Distribution', 'EMS', 'ADES', 'CWL', 'OGC'] }, -}) + 'stats': { + 'method': '.*', + 'route': "/weaver/.*" + }, + 'redirect': { + 'doc': 'https://pavics-weaver.readthedocs.io/', + 'releasenotes': 'https://github.com/crim-ca/weaver/blob/master/CHANGES.rst', + 'support': 'https://github.com/crim-ca/weaver/issues', + 'source': 'https://github.com/crim-ca/weaver', + 'tryme': 'https://${PAVICS_FQDN_PUBLIC}/weaver/', + 'licence': 'https://github.com/crim-ca/weaver/blob/${WEAVER_VERSION}/LICENSE.txt', + 'provenance': 'https://github.com/crim-ca/weaver' + }, + "monitoring": { + "Weaver": { + 'request': { + 'url': 'http://weaver:4001/' + }, + 'response': { + 'text': '\{.*"message":.*"Weaver Information".*\}' + } + } + } +} # vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/components/weaver/config/canarie-api/docker-compose-extra.yml b/birdhouse/components/weaver/config/canarie-api/docker-compose-extra.yml index d88cd4899..e441d76eb 100644 --- a/birdhouse/components/weaver/config/canarie-api/docker-compose-extra.yml +++ b/birdhouse/components/weaver/config/canarie-api/docker-compose-extra.yml @@ -4,4 +4,11 @@ services: # extend proxy configuration with weaver endpoints proxy: volumes: + # NOTE: + # Prefix '0_' to the mounted file name to ensure it is loaded first by 'birdhouse/config/canarie-api/docker_configuration.py' + # This ensures that the alphabetical loading order it defines will first load the default configs, then will load + # the weaver config, and then all python configuration within 'optional-components/canarie-api-full-monitoring'. + # It is important to load Weaver before, as the full monitoring need to know if 'Weaver' service is added to the + # stack in order to apply the corresponding public endpoint conditionally. + - ./components/weaver/config/canarie-api/canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/0_weaver_config.py:ro - ./components/weaver/config/canarie-api/canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/weaver_canarie_api_monitoring.py:ro diff --git a/birdhouse/config/canarie-api/default.env b/birdhouse/config/canarie-api/default.env index 608ecde8b..938286617 100644 --- a/birdhouse/config/canarie-api/default.env +++ b/birdhouse/config/canarie-api/default.env @@ -1,10 +1,9 @@ # Folder inside "proxy" container to drop extra monitoring config export CANARIE_MONITORING_EXTRA_CONF_DIR="/conf.d" -export PROXY_IMAGE="pavics/canarieapi:0.3.5" +export PROXY_IMAGE="pavics/canarieapi:0.7.1" # add any component that this component requires to run COMPONENT_DEPENDENCIES=" ./config/proxy " - diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index 9f4d7d40d..a52ba59b8 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -1,7 +1,13 @@ # coding: utf-8 +import datetime import os import logging +# note: +# Can only be dependencies also available in CanarieAPI docker image. +# Otherwise, they must be installed after the fact with the entrypoint script. +import requests_cache # see entrypoint script + logger = logging.getLogger("canarie-api-config") MY_SERVER_NAME = 'https://${PAVICS_FQDN_PUBLIC}/canarie' @@ -12,13 +18,99 @@ DATABASE = { 'log_pid': '/var/run/nginx.pid' } + +# Below is to avoid rate-limiting as much as possible on multiple runs. +# Because the logparser and monitoring must load the application context and configuration, +# the requests performed to retrieve version numbers and release times occurs on every cron interval. +REQUEST_CACHE = "/tmp/canarieapi-config-cache" +os.makedirs(REQUEST_CACHE, exist_ok=True) +SESSION = requests_cache.CachedSession( + "canarieapi-config-cache", + use_temp=True, + backend="filesystem", + expire_after=datetime.timedelta(days=1), + cache_control=False, # prefer our request cache options, not ones provided by responses + stale_if_error=True, + allowable_codes=[200], + allowable_methods=["GET"], +) + + +def _get_release_time_from_github_tag(repository_reference: str, tagged_version: str) -> str: + repo = repository_reference.split("github.com/")[-1] + url = f"https://api.github.com/repos/{repo}/tags" + resp = SESSION.get(url, headers={"Accept": "application/json"}) + if resp.status_code != 200: + raise ValueError( + f"HTTP status from tags list: {resp.status_code} from [{url}] with detail:\n{resp.text!s}" + ) + + for tag_info in resp.json(): + tag_ver = tag_info.get("name") + tag_commit_url = tag_info.get("commit", {}).get("url") + if tag_ver == tagged_version or tag_ver == "v" + tagged_version and tag_commit_url: + break + else: + raise ValueError(f"No matching version in tagged releases under [{url}].") + + resp = SESSION.get(tag_commit_url, headers={"Accept": "application/json"}) + if resp.status_code != 200: + raise ValueError( + f"HTTP status from tag info: {resp.status_code} from [{tag_commit_url}] with detail:\n{resp.text!s}" + ) + + commit = resp.json()["commit"] + commit_date = commit["committer"]["date"] + return commit_date + + +def _get_release_time_from_docker_tag(repository_reference: str, tagged_version: str) -> str: + ns, repo = repository_reference.split("/", 1) + url = f"https://hub.docker.com/v2/namespaces/{ns}/repositories/{repo}/tags/{tagged_version}" + resp = SESSION.get(url, headers={"Accept": "application/json"}) + if resp.status_code != 200: + raise ValueError( + f"HTTP status from tag info: {resp.status_code} from [{url}] with detail:\n{resp.text!s}" + ) + + updated = resp.json()["last_updated"] + return updated + + +def get_release_time_from_repo_tag(repository_type: str, repository_reference: str, tagged_version: str) -> str: + try: + if repository_type == "github": + release = _get_release_time_from_github_tag(repository_reference, tagged_version) + elif repository_type == "docker": + release = _get_release_time_from_docker_tag(repository_reference, tagged_version) + else: + raise ValueError(f"Unknown repository parsing type: [{repository_type}]") + if release: + logger.error("canarie-api: found version [%s] release time [%s] from [%s: %s]", + tagged_version, release, repository_type, repository_reference) + return release + raise ValueError("no release time value found") + except Exception as exc: + logger.error("canarie-api: failed retrieving version [%s] release time from [%s: %s]", + tagged_version, repository_type, repository_reference, exc_info=exc) + return "unknown" + + SERVICES = { - 'node': { + 'node': { # partial duplicate of first item from 'PLATFORM' for backward compatibility 'info': { 'name': 'Node', - 'synopsis': 'Nodes are data, compute and index endpoints accessed through the PAVICS platform or external clients. The Node service is the backend that allows: data storage, harvesting, indexation and discovery of local and federated data; authentication and authorization; server registration and management. Node service is therefore composed of several other services.', - 'version': '1.25.7', - 'releaseTime': '2023-04-20T14:15:34Z', + 'synopsis': ( + 'Nodes are data, compute and index endpoints accessed through the platform or external clients. ' + 'The Node service is the backend that allows: data storage, harvesting, indexation and discovery of ' + 'local and federated data; authentication and authorization; server registration and management. ' + 'Node service is therefore composed of several other services.' + ), + # NOTE: + # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. + # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. + 'version': '1.26.0', + 'releaseTime': '2023-04-20T20:42:31Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -27,28 +119,31 @@ SERVICES = { }, 'stats': { 'method': '.*', - 'route': '(?!)' # this will be set by CANARIE_STATS_ROUTES (see below) + 'route': '(?!)' # this will be set by CANARIE_STATS_ROUTES (see below) }, 'redirect': { 'doc': 'https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html', - 'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/releases', + 'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md', 'support': 'https://github.com/bird-house/birdhouse-deploy/issues', 'source': 'https://github.com/bird-house/birdhouse-deploy', 'tryme': 'https://${PAVICS_FQDN_PUBLIC}', 'licence': 'https://pavics-sdi.readthedocs.io/en/latest/license.html', 'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html' }, - 'monitoring': {} + 'monitoring': {} # filled in after processing everything, see end of script } } PLATFORMS = { - 'pavics': { + 'server': { 'info': { - 'name': 'PAVICS', - 'synopsis': 'The PAVICS (Power Analytics for Visualization of Climate Science) platform is a collection of climate analysis services served through Open Geospatial Consortium (OGC) protocols. These services include data access, processing and visualization. Both data and algorithms can be accessed either programmatically, through OGC-compliant clients such as QGIS or ArcGIS, or a custom web interface.', - 'version': '1.25.7', - 'releaseTime': '2023-04-20T14:15:34Z', + 'name': '${SERVER_NAME}', + 'synopsis': """${SERVER_DESCRIPTION}""".replace("\n", " ").strip(), + # NOTE: + # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. + # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. + 'version': '1.26.0', + 'releaseTime': '2023-04-20T20:42:31Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -56,7 +151,7 @@ PLATFORMS = { }, 'stats': { 'method': '.*', - 'route': '(?!)' # this can be set by individual components (eg: the legacy project-api component) + 'route': '(?!)' # this can be set by individual components (eg: the legacy project-api component) }, 'redirect': { 'doc': 'https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html', @@ -73,20 +168,56 @@ PLATFORMS = { } CANARIE_MONITORING_EXTRA_CONF_DIR = os.environ.get( - 'CANARIE_MONITORING_EXTRA_CONF_DIR', '/bogus-notexist') + 'CANARIE_MONITORING_EXTRA_CONF_DIR', '/bogus-notexist' +) + -CANARIE_STATS_ROUTES=[] +def exec_file(filepath, **kwargs): + """ + Backward compatible ``execfile`` equivalent for Python 3.x that allows keyword arguments. + + See https://stackoverflow.com/a/41658338/5936364. + """ + _globals = kwargs.get("globals") or globals() + _locals = kwargs.get("locals") or _globals # as per exec/execfile documentation + _globals.update({ + "__file__": filepath, + "__name__": "__main__", + }) + with open(filepath, 'rb') as file: + exec(compile(file.read(), filepath, "exec"), _globals, _locals) + + +CANARIE_STATS_ROUTES = [] if os.path.exists(CANARIE_MONITORING_EXTRA_CONF_DIR): # alphabetically sorted for reproducible override precedence for extra_conf in sorted(os.listdir(CANARIE_MONITORING_EXTRA_CONF_DIR)): - extra_conf_full_path = "{CANARIE_MONITORING_EXTRA_CONF_DIR}/{extra_conf}".format(**locals()) + extra_conf_full_path = f"{CANARIE_MONITORING_EXTRA_CONF_DIR}/{extra_conf}" # only handle files ending with .py if os.path.isfile(extra_conf_full_path) and extra_conf_full_path.endswith(".py"): - logger.info("canarie-api: loading extra config '{extra_conf_full_path}'".format(**locals())) - execfile(extra_conf_full_path) + logger.info("canarie-api: loading extra config '%s'", extra_conf_full_path) + exec_file(extra_conf_full_path, locals=locals()) else: - logger.info("canarie-api: ignoring extra config '{extra_conf_full_path}'".format(**locals())) + logger.info("canarie-api: ignoring extra config '%s'", extra_conf_full_path) + + +# For historical reasons, the "node" service was used to contain all "services" of the current "platform". +# However, those services should instead be listed directly at the root as individual services of the current platform, +# to directly obtain their status instead of going through the "node" service sub-page. This also allows us to provide +# the individual links (docs, release, etc.) for the services themselves. The services can also be represented nested +# under the "Platform" section as well. This "Platform" section could be extended with other known platforms part of a +# DACCS network. For backward-compatibility, repopulate the "node" service using moved definitions. +_NODE = SERVICES["node"]["monitoring"] +_PLATFORM = PLATFORMS["server"]["monitoring"] +for svc_name, svc_config in SERVICES.items(): + if svc_name == "node": + continue + for mon_name, mon_config in svc_config["monitoring"].items(): + _NODE.setdefault(mon_name, mon_config) + _PLATFORM.setdefault(mon_name, mon_config) # type: ignore if CANARIE_STATS_ROUTES: SERVICES['node']['stats']['route'] = '/(' + '|'.join(r.strip('/') for r in CANARIE_STATS_ROUTES) + ')/.*' + +# vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/config/canarie-api/entrypoint b/birdhouse/config/canarie-api/entrypoint index 8dd243c4f..93155ab77 100755 --- a/birdhouse/config/canarie-api/entrypoint +++ b/birdhouse/config/canarie-api/entrypoint @@ -5,5 +5,12 @@ # remove if cause problem with newer version of canarieapi touch /etc/cron.d/canarieapi-cron +pip install requests-cache + # chain existing cmd -exec /bin/sh -c "env >> /etc/environment && cron && nginx && gunicorn -b 0.0.0.0:2000 --workers 1 --log-level=DEBUG --timeout 30 -k gevent wsgi" +exec /bin/sh -c "\ + env >> /etc/environment && \ + cron && \ + nginx && \ + gunicorn -b 0.0.0.0:2000 --workers 1 --log-level=DEBUG --timeout 30 -k gevent canarieapi.wsgi \ +" diff --git a/birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py.template index 1def70114..d172d058a 100644 --- a/birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/catalog/config/canarie-api/canarie_api_monitoring.py.template @@ -1,5 +1,34 @@ -SERVICES['node']['monitoring']['Catalog'] = { - 'request': { - 'url': 'http://${PAVICS_FQDN}:8086/pywps?service=WPS&version=1.0.0&request=GetCapabilities' +SERVICES['Catalog'] = { + 'info': { + 'name': 'Catalog', + 'synopsis': 'Catalog', + 'version': "${CATALOG_VERSION}", + 'institution': 'Ouranos', + 'releaseTime': get_release_time_from_repo_tag("docker", "${CATALOG_DOCKER}", "${CATALOG_VERSION}"), + 'researchSubject': 'Climate', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Catalog', + 'tags': ['Catalog', 'Data'] + }, + 'stats': { + 'method': '.*', + 'route': 'http://${PAVICS_FQDN}:8086/.*' + }, + 'redirect': { + 'doc': 'https://ouranosinc.github.io/pavics-sdi/arch/data_catalog.html#pavics-datacatalog', + 'releasenotes': 'https://github.com/Ouranosinc/PAVICS-DataCatalog/blob/master/CHANGES.md', + 'support': 'https://github.com/ouranosinc/PAVICS-DataCatalog/issues', + 'source': 'https://github.com/ouranosinc/PAVICS-DataCatalog', + 'tryme': 'http://${PAVICS_FQDN}:8086/pywps?service=WPS&version=1.0.0&request=GetCapabilities', + 'licence': 'https://ouranosinc.github.io/pavics-sdi/arch/data_catalog.html#credits', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/arch/data_catalog.html#pavics-datacatalog' + }, + "monitoring": { + "Catalog": { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://${PAVICS_FQDN}:8086/pywps?service=WPS&version=1.0.0&request=GetCapabilities' + } + } } } diff --git a/birdhouse/config/catalog/default.env b/birdhouse/config/catalog/default.env index 5409945fa..baaa58bc1 100644 --- a/birdhouse/config/catalog/default.env +++ b/birdhouse/config/catalog/default.env @@ -4,6 +4,9 @@ # must use single quotes to avoid early expansion before overrides in env.local # are applied and must be added to the list of DELAYED_EVAL. +export CATALOG_DOCKER=pavics/pavics-datacatalog +export CATALOG_VERSION=0.6.11 + # This tells the postgres service to create a database on startup export POSTGRES_DATABASES_TO_CREATE=" $POSTGRES_DATABASES_TO_CREATE @@ -24,6 +27,12 @@ VARS=" \$CATALOG_THREDDS_SERVICE " +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$CATALOG_DOCKER + \$CATALOG_VERSION +" + # add any component that this component requires to run COMPONENT_DEPENDENCIES=" ./config/postgres diff --git a/birdhouse/config/catalog/docker-compose-extra.yml b/birdhouse/config/catalog/docker-compose-extra.yml index a448f7943..8825c0b53 100644 --- a/birdhouse/config/catalog/docker-compose-extra.yml +++ b/birdhouse/config/catalog/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: catalog: - image: pavics/pavics-datacatalog:0.6.11 + image: ${CATALOG_DOCKER}:${CATALOG_VERSION} container_name: catalog ports: - "8086:80" diff --git a/birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py.template index 658229de3..043ad439b 100644 --- a/birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/finch/config/canarie-api/canarie_api_monitoring.py.template @@ -1,15 +1,18 @@ -SERVICES['node']['monitoring']['Finch'] = { - 'request': { - 'url': 'http://${PAVICS_FQDN}:8095/wps?service=WPS&version=1.0.0&request=GetCapabilities' - } -} +# used more than once, avoid unnecessary requests +FINCH_VERSION = "${FINCH_VERSION}" +FINCH_RELEASE_TIME = get_release_time_from_repo_tag("github", "bird-house/finch", FINCH_VERSION) + SERVICES['indices'] = { 'info': { 'name': 'Climate indices', - 'synopsis': 'Users of climate data are interested in specific indices such as the number of freeze-thaw cycles, the number of degree-days of cooling, the duration of heatwaves, etc. This returns annual values of the most popular climate indices.', - 'version': '0.7.1', + 'synopsis': ( + 'Users of climate data are interested in specific indices such as the number of freeze-thaw cycles, ' + 'the number of degree-days of cooling, the duration of heatwaves, etc. ' + 'This returns annual values of the most popular climate indices.' + ), + 'version': FINCH_VERSION, 'institution': 'Ouranos', - 'releaseTime': '2021-03-25T00:00:00Z', + 'releaseTime': FINCH_RELEASE_TIME, 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', 'category': 'Data Manipulation', @@ -39,7 +42,7 @@ SERVICES['indices'] = { SERVICES['slicer'] = { 'info': { 'name': 'Spatial and temporal data slicer', - 'synopsis': 'These services allows the subsetting and download of NetCDF over user-defined regions and periods.', + 'synopsis': 'These services allow the subsetting and download of NetCDF over user-defined regions and periods.', 'version': '0.7.1', 'institution': 'Ouranos', 'releaseTime': '2021-03-25T00:00:00Z', @@ -64,8 +67,48 @@ SERVICES['slicer'] = { 'monitoring': { 'Finch': { 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&version=1.0.0&request=GetCapabilities' + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://${PAVICS_FQDN}:8095/?service=WPS&version=1.0.0&request=GetCapabilities' } }, } } + +SERVICES['Finch'] = { + 'info': { + 'name': 'Finch', + 'synopsis': ( + 'Users of climate data are interested in specific indices such as the number of freeze-thaw cycles, ' + 'the number of degree-days of cooling, the duration of heatwaves, etc. This returns annual values of ' + 'the most popular climate indices.' + ), + 'version': FINCH_VERSION, + 'institution': 'Ouranos', + 'releaseTime': FINCH_RELEASE_TIME, + 'researchSubject': 'Climatology', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Data Manipulation', + 'tags': ['Climatology', 'Cloud'] + }, + 'stats': { + 'method': '.*', + 'route': '${TWITCHER_PROTECTED_PATH}/finch.*' + }, + 'redirect': { + 'doc': 'https://finch.readthedocs.io/en/latest/index.html', + 'releasenotes': 'https://github.com/bird-house/finch/releases', + 'support': 'https://github.com/bird-house/finch/issues', + 'source': 'https://github.com/bird-house/finch', + 'tryme': 'https://ouranosinc.github.io/pavics-sdi/notebooks/climate_indices.html', + 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + 'monitoring': { + 'Finch': { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://${PAVICS_FQDN}:8095/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, + }, +} diff --git a/birdhouse/config/finch/default.env b/birdhouse/config/finch/default.env index 51e7a4968..8bdaafc08 100644 --- a/birdhouse/config/finch/default.env +++ b/birdhouse/config/finch/default.env @@ -4,11 +4,8 @@ # must use single quotes to avoid early expansion before overrides in env.local # are applied and must be added to the list of DELAYED_EVAL. -export FINCH_IMAGE="birdhouse/finch:version-0.9.2" - -# When canarie-api is monitoring twitcher, at least one WPS service protected by twitcher -# needs to be available. This sets this component as the service used to monitor twitcher. -export WPS_FOR_TWITCHER_MONITORING=${WPS_FOR_TWITCHER_MONITORING:-finch} +export FINCH_VERSION=0.9.2 +export FINCH_IMAGE="birdhouse/finch:version-${FINCH_VERSION}" # This tells the postgres service to create a database on startup export POSTGRES_DATABASES_TO_CREATE=" @@ -27,3 +24,8 @@ COMPONENT_DEPENDENCIES=" ./config/wps_outputs-volume ./config/postgres " + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$FINCH_VERSION +" diff --git a/birdhouse/config/flyingpigeon/.gitignore b/birdhouse/config/flyingpigeon/.gitignore index d25d278a4..6156a69ce 100644 --- a/birdhouse/config/flyingpigeon/.gitignore +++ b/birdhouse/config/flyingpigeon/.gitignore @@ -1,3 +1,4 @@ wps.cfg +config/canarie-api/canarie_api_monitoring.py config/magpie/permissions.cfg config/magpie/providers.cfg diff --git a/birdhouse/config/flyingpigeon/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/flyingpigeon/config/canarie-api/canarie_api_monitoring.py.template new file mode 100644 index 000000000..6af7a86c8 --- /dev/null +++ b/birdhouse/config/flyingpigeon/config/canarie-api/canarie_api_monitoring.py.template @@ -0,0 +1,39 @@ +FLYINGPIGEON_VERSION = "${FLYINGPIGEON_VERSION}" +FLYINGPIGEON_RELEASE = get_release_time_from_repo_tag("github", "bird-house/flyingpigeon", FLYINGPIGEON_VERSION) + +SERVICES['flyingpigeon'] = { + 'info': { + 'name': 'Climatology modeling and analysis', + 'synopsis': ( + 'WPS processes for climate model data, indices and extreme events.' + ), + 'version': FLYINGPIGEON_VERSION, + 'institution': 'bird-house', + 'releaseTime': FLYINGPIGEON_RELEASE, + 'researchSubject': 'Climatology', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Processing', + 'tags': ['Climatology', 'WPS', 'OGC'], + }, + 'stats': { + 'method': '.*', + 'route': '${TWITCHER_PROTECTED_PATH}/flyingpigeon.*' + }, + 'redirect': { + 'doc': 'https://flyingpigeon.readthedocs.io/', + 'releasenotes': 'https://github.com/bird-house/flyingpigeon/blob/master/CHANGES.rst', + 'support': 'https://github.com/bird-house/flyingpigeon/issues', + 'source': 'https://github.com/bird-house/flyingpigeon', + 'tryme': 'http://${PAVICS_FQDN}:8093/wps?service=WPS&version=1.0.0&request=GetCapabilities', + 'licence': 'https://github.com/bird-house/flyingpigeon/blob/master/LICENSE.txt', + 'provenance': 'https://github.com/bird-house/flyingpigeon' + }, + 'monitoring': { + 'Flyingpigeon': { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://${PAVICS_FQDN}:8093/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, + } +} diff --git a/birdhouse/config/flyingpigeon/config/canarie-api/docker-compose-extra.yml b/birdhouse/config/flyingpigeon/config/canarie-api/docker-compose-extra.yml new file mode 100644 index 000000000..1f45a13be --- /dev/null +++ b/birdhouse/config/flyingpigeon/config/canarie-api/docker-compose-extra.yml @@ -0,0 +1,5 @@ +version: "3.4" +services: + proxy: + volumes: + - ./config/flyingpigeon/config/canarie-api/canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/flyingpigeon_canarie_api_monitoring.py:ro diff --git a/birdhouse/config/flyingpigeon/default.env b/birdhouse/config/flyingpigeon/default.env index 333187d78..4a842692d 100644 --- a/birdhouse/config/flyingpigeon/default.env +++ b/birdhouse/config/flyingpigeon/default.env @@ -1,7 +1,4 @@ - -# When canarie-api is monitoring twitcher, at least one WPS service protected by twitcher -# needs to be available. This sets this component as the service used to monitor twitcher. -export WPS_FOR_TWITCHER_MONITORING=${WPS_FOR_TWITCHER_MONITORING:-flyingpigeon} +export FLYINGPIGEON_VERSION=1.6 # This tells the postgres service to create a database on startup export POSTGRES_DATABASES_TO_CREATE=" @@ -20,3 +17,8 @@ COMPONENT_DEPENDENCIES=" ./config/postgres ./config/wps_outputs-volume " + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$FLYINGPIGEON_VERSION +" diff --git a/birdhouse/config/flyingpigeon/docker-compose-extra.yml b/birdhouse/config/flyingpigeon/docker-compose-extra.yml index 5a2fc259f..6e1a0c043 100644 --- a/birdhouse/config/flyingpigeon/docker-compose-extra.yml +++ b/birdhouse/config/flyingpigeon/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: flyingpigeon: - image: birdhouse/flyingpigeon:1.6 + image: birdhouse/flyingpigeon:${FLYINGPIGEON_VERSION} container_name: flyingpigeon environment: - PYWPS_CFG=/wps.cfg diff --git a/birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py.template index ed91df6e6..4c5c0299d 100644 --- a/birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/frontend/config/canarie-api/canarie_api_monitoring.py.template @@ -1,4 +1,4 @@ -PLATFORMS['pavics']['monitoring']['Frontend'] = { +PLATFORMS['server']['monitoring']['Frontend'] = { 'request': { 'url': 'https://${PAVICS_FQDN_PUBLIC}' } diff --git a/birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py.template index 2c2b89e1f..67fe4d41f 100644 --- a/birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/geoserver/config/canarie-api/canarie_api_monitoring.py.template @@ -1,6 +1,40 @@ -SERVICES['node']['monitoring'].update({'Geoserver': { - 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}/geoserver/web/' +SERVICES['GeoServer'] = { + 'info': { + 'name': 'GeoServer', + 'synopsis': ( + 'GeoServer is the reference implementation of the Open Geospatial Consortium (OGC) ' + 'Web Feature Service (WFS) and Web Coverage Service (WCS) standards, as well as a high performance ' + 'certified compliant Web Map Service (WMS), compliant Catalog Service for the Web (CSW) and ' + 'implementing Web Processing Service (WPS). GeoServer forms a core component of the Geospatial Web.' + ), + 'version': "${GEOSERVER_VERSION}", + 'institution': 'Ouranos', + 'releaseTime': get_release_time_from_repo_tag("docker", "${GEOSERVER_DOCKER}", "${GEOSERVER_TAGGED}"), + 'researchSubject': 'Geospatial', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Data Catalog', + 'tags': ['Data', 'Geospatial', 'Catalog', 'OGC', 'WFS', 'WMS', 'WPS'] + }, + 'stats': { + 'method': '.*', + 'route': "/geoserver/.*" + }, + 'redirect': { + 'doc': 'https://docs.geoserver.org/', + 'releasenotes': 'https://geoserver.org/release/${GEOSERVER_VERSION}/', + 'support': 'https://github.com/kartoza/docker-geoserver/issues', + 'source': 'https://github.com/kartoza/docker-geoserver', + 'tryme': 'https://${PAVICS_FQDN_PUBLIC}/geoserver/', + 'licence': 'https://github.com/geoserver/geoserver/blob/${GEOSERVER_VERSION}/LICENSE.txt', + 'provenance': 'https://github.com/kartoza/docker-geoserver' + }, + "monitoring": { + "GeoServer": { + 'request': { + 'url': 'https://${PAVICS_FQDN_PUBLIC}/geoserver/web/' + } + } } -}}) +} + CANARIE_STATS_ROUTES.append('geoserver') diff --git a/birdhouse/config/geoserver/default.env b/birdhouse/config/geoserver/default.env index 51aa946b1..99551ce91 100644 --- a/birdhouse/config/geoserver/default.env +++ b/birdhouse/config/geoserver/default.env @@ -7,7 +7,11 @@ # Cache kartoza/geoserver docker build on pavics org since their tags are # "moving" tags, meaning not reproducible behavior ! # See https://github.com/kartoza/docker-geoserver/issues/232#issuecomment-808754831 -export GEOSERVER_IMAGE="pavics/geoserver:2.22.2-kartoza-build20230226-r7-allow-change-context-root-and-fix-missing-stable-plugins-and-avoid-chown-datadir" +# The version is used for representation in CanarieAPI, while the full tag is used to reference the image. +export GEOSERVER_DOCKER=pavics/geoserver +export GEOSERVER_VERSION=2.22.2 +export GEOSERVER_TAGGED=2.22.2-kartoza-build20230226-r7-allow-change-context-root-and-fix-missing-stable-plugins-and-avoid-chown-datadir +export GEOSERVER_IMAGE="${GEOSERVER_DOCKER}:${GEOSERVER_TAGGED}" export GEOSERVER_ADMIN_USER="admin" @@ -33,3 +37,11 @@ VARS=" \$GEOSERVER_ADMIN_USER \$GEOSERVER_ADMIN_PASSWORD " + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$GEOSERVER_DOCKER + \$GEOSERVER_VERSION + \$GEOSERVER_TAGGED + \$GEOSERVER_IMAGE +" diff --git a/birdhouse/config/hummingbird/.gitignore b/birdhouse/config/hummingbird/.gitignore index 1d94a50e6..02da491b5 100644 --- a/birdhouse/config/hummingbird/.gitignore +++ b/birdhouse/config/hummingbird/.gitignore @@ -1,2 +1,3 @@ +config/canarie-api/canarie_api_monitoring.py config/magpie/providers.cfg custom.cfg diff --git a/birdhouse/config/hummingbird/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/hummingbird/config/canarie-api/canarie_api_monitoring.py.template new file mode 100644 index 000000000..fdcccb47f --- /dev/null +++ b/birdhouse/config/hummingbird/config/canarie-api/canarie_api_monitoring.py.template @@ -0,0 +1,39 @@ +HUMMINGBIRD_VERSION = "${HUMMINGBIRD_VERSION}" +HUMMINGBIRD_RELEASE = get_release_time_from_repo_tag("docker", "pavics/hummingbird", HUMMINGBIRD_VERSION) + +SERVICES['hummingbird'] = { + 'info': { + 'name': 'Climatology compliance checker.', + 'synopsis': ( + 'A Web Processing Service for compliance checks used in the climate science community.' + ), + 'version': HUMMINGBIRD_VERSION, + 'institution': 'bird-house', + 'releaseTime': HUMMINGBIRD_RELEASE, + 'researchSubject': 'Climatology', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Processing', + 'tags': ['Climatology', 'Checker', 'Compliance', 'CF-conventions', 'WPS', 'OGC'], + }, + 'stats': { + 'method': '.*', + 'route': '${TWITCHER_PROTECTED_PATH}/hummingbird.*' + }, + 'redirect': { + 'doc': 'https://hummingbird.readthedocs.io/', + 'releasenotes': 'https://github.com/bird-house/hummingbird/blob/master/CHANGES.rst', + 'support': 'https://github.com/bird-house/hummingbird/issues', + 'source': 'https://github.com/bird-house/hummingbird', + 'tryme': 'http://${PAVICS_FQDN}:8097/wps?service=WPS&version=1.0.0&request=GetCapabilities', + 'licence': 'https://github.com/bird-house/hummingbird/blob/master/LICENSE.txt', + 'provenance': 'https://github.com/bird-house/hummingbird' + }, + 'monitoring': { + 'Hummingbird': { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://${PAVICS_FQDN}:8097/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, + } +} diff --git a/birdhouse/config/hummingbird/config/canarie-api/docker-compose-extra.yml b/birdhouse/config/hummingbird/config/canarie-api/docker-compose-extra.yml new file mode 100644 index 000000000..4b1c29405 --- /dev/null +++ b/birdhouse/config/hummingbird/config/canarie-api/docker-compose-extra.yml @@ -0,0 +1,5 @@ +version: "3.4" +services: + proxy: + volumes: + - ./config/hummingbird/config/canarie-api/canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/hummingbird_canarie_api_monitoring.py:ro diff --git a/birdhouse/config/hummingbird/default.env b/birdhouse/config/hummingbird/default.env index f2afc8f27..f45f8e8d6 100644 --- a/birdhouse/config/hummingbird/default.env +++ b/birdhouse/config/hummingbird/default.env @@ -1,6 +1,4 @@ -# When canarie-api is monitoring twitcher, at least one WPS service protected by twitcher -# needs to be available. This sets this component as the service used to monitor twitcher. -export WPS_FOR_TWITCHER_MONITORING=${WPS_FOR_TWITCHER_MONITORING:-hummingbird} +export HUMMINGBIRD_VERSION=0.5_dev # This tells the postgres service to create a database on startup export POSTGRES_DATABASES_TO_CREATE=" @@ -20,3 +18,8 @@ COMPONENT_DEPENDENCIES=" ./config/wps_outputs-volume ./config/data-volume " + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$HUMMINGBIRD_VERSION +" diff --git a/birdhouse/config/hummingbird/docker-compose-extra.yml b/birdhouse/config/hummingbird/docker-compose-extra.yml index edfa2469d..36a316d38 100644 --- a/birdhouse/config/hummingbird/docker-compose-extra.yml +++ b/birdhouse/config/hummingbird/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: hummingbird: - image: pavics/hummingbird:0.5_dev + image: pavics/hummingbird:${HUMMINGBIRD_VERSION} container_name: hummingbird environment: HOSTNAME: $HOSTNAME diff --git a/birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template index 51eb6f5f5..18af01348 100644 --- a/birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/jupyterhub/config/canarie-api/canarie_api_monitoring.py.template @@ -1,7 +1,35 @@ -SERVICES['node']['monitoring']['Jupyter'] = { - 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}/jupyter/hub/login' +SERVICES['Jupyter'] = { + 'info': { + 'name': 'Jupyter', + 'synopsis': 'Jupyter notebooks portal.', + 'version': "${JUPYTER_VERSION}", + 'releaseTime': get_release_time_from_repo_tag("docker", "${JUPYTERHUB_DOCKER}", "${JUPYTERHUB_VERSION}"), + 'institution': 'Ouranos', + 'researchSubject': 'Any', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Research', + 'tags': ['Development', 'Research', 'Notebooks'] + }, + 'stats': { + 'method': '.*', + 'route': '/jupyter/.*' + }, + 'redirect': { + 'doc': 'https://jupyter.org/hub', + 'releasenotes': 'https://github.com/Ouranosinc/jupyterhub/tags', # no CHANGES file available + 'support': 'https://github.com/Ouranosinc/jupyterhub/issues', + 'source': 'https://github.com/Ouranosinc/jupyterhub', + 'tryme': 'https://${PAVICS_FQDN_PUBLIC}/jupyter/', + 'licence': 'https://github.com/Ouranosinc/jupyterhub/blob/${JUPYTERHUB_VERSION}/LICENSE', + 'provenance': '' + }, + "monitoring": { + "Jupyter": { + 'request': { + 'url': 'https://${PAVICS_FQDN_PUBLIC}/jupyter/hub/login' + }, + } } } -CANARIE_STATS_ROUTES.append('jupyter') +CANARIE_STATS_ROUTES.append('jupyter') diff --git a/birdhouse/config/jupyterhub/default.env b/birdhouse/config/jupyterhub/default.env index 8bd2e80e7..7543b4733 100644 --- a/birdhouse/config/jupyterhub/default.env +++ b/birdhouse/config/jupyterhub/default.env @@ -4,6 +4,9 @@ # must use single quotes to avoid early expansion before overrides in env.local # are applied and must be added to the list of DELAYED_EVAL. +export JUPYTERHUB_DOCKER=pavics/jupyterhub +export JUPYTERHUB_VERSION=1.4.0-20210506 + # Jupyter single-user server images, can be overriden in env.local to have a space separated list of multiple images export DOCKER_NOTEBOOK_IMAGES="pavics/workflow-tests:221130-update230403" @@ -63,6 +66,8 @@ OPTIONAL_VARS=" \$JUPYTER_LOGIN_BANNER_BOTTOM_SECTION \$JUPYTER_LOGIN_TERMS_URL \$JUPYTERHUB_CONFIG_OVERRIDE + \$JUPYTERHUB_DOCKER + \$JUPYTERHUB_VERSION " # add any component that this component requires to run diff --git a/birdhouse/config/jupyterhub/docker-compose-extra.yml b/birdhouse/config/jupyterhub/docker-compose-extra.yml index 03cf5f262..bc608b0bd 100644 --- a/birdhouse/config/jupyterhub/docker-compose-extra.yml +++ b/birdhouse/config/jupyterhub/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: jupyterhub: - image: pavics/jupyterhub:1.4.0-20210506 + image: ${JUPYTERHUB_DOCKER}:${JUPYTERHUB_VERSION} container_name: jupyterhub hostname: jupyterhub ports: diff --git a/birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py.template index ff77a2b7b..60b030b38 100644 --- a/birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/magpie/config/canarie-api/canarie_api_monitoring.py.template @@ -1,9 +1,42 @@ -SERVICES['node']['monitoring'].update({'Magpie': { - 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}/magpie/version' +SERVICES['Magpie'] = { + 'info': { + 'name': 'Magpie', + 'synopsis': ( + 'Magpie is service for AuthN/AuthZ accessible via a REST API. ' + 'It allows you to manage User/Group/Service/Resource/Permission management ' + 'and integrates with Twitcher.' + ), + 'version': "${MAGPIE_VERSION}", + 'institution': 'Ouranos', + 'releaseTime': get_release_time_from_repo_tag("github", "Ouranosinc/Magpie", "${MAGPIE_VERSION}"), + 'researchSubject': 'Security', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Security', + 'tags': ['Security', 'Management', 'Access', 'Policy Decision Point'] }, - 'response': { - 'text': '\{.*"code": 200.*"type": "application/json".*\}' + 'stats': { + 'method': '.*', + 'route': "/magpie/.*" + }, + 'redirect': { + 'doc': 'https://pavics-magpie.readthedocs.io/', + 'releasenotes': 'https://github.com/Ouranosinc/Magpie/blob/master/CHANGES.rst', + 'support': 'https://github.com/Ouranosinc/Magpie/issues', + 'source': 'https://github.com/Ouranosinc/Magpie', + 'tryme': 'https://${PAVICS_FQDN_PUBLIC}/magpie/', + 'licence': 'https://github.com/Ouranosinc/Magpie/blob/${MAGPIE_VERSION}/LICENSE', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + "monitoring": { + "Magpie": { + 'request': { + 'url': 'https://${PAVICS_FQDN_PUBLIC}/magpie/version' + }, + 'response': { + 'text': r'\{.*"code": 200.*"type": "application/json".*\}' + } + } } -}}) +} + CANARIE_STATS_ROUTES.append('magpie') diff --git a/birdhouse/config/magpie/default.env b/birdhouse/config/magpie/default.env index 4ebb21c44..3a9c7f4e0 100644 --- a/birdhouse/config/magpie/default.env +++ b/birdhouse/config/magpie/default.env @@ -69,4 +69,5 @@ OPTIONAL_VARS=" \$MAGPIE_SMTP_SSL \$MAGPIE_SMTP_PASSWORD \$MAGPIE_LOG_LEVEL + \$MAGPIE_VERSION " diff --git a/birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py.template index 9dde14cb8..84eae18e3 100644 --- a/birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/malleefowl/config/canarie-api/canarie_api_monitoring.py.template @@ -1,5 +1,34 @@ -SERVICES['node']['monitoring']['Malleefowl'] = { - 'request': { - 'url': 'http://${PAVICS_FQDN}:8091/wps?service=WPS&version=1.0.0&request=GetCapabilities' +SERVICES['Malleefowl'] = { + 'info': { + 'name': 'Malleefowl', + 'synopsis': 'A Web Processing Service for Climate Data Access and Workflows.', + 'version': "${MALLEEFOWL_VERSION}", + 'releaseTime': get_release_time_from_repo_tag("github", "Ouranosinc/malleefowl", "${MALLEEFOWL_VERSION}"), + 'institution': 'Ouranos', + 'researchSubject': 'Climatology', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Resource/Cloud Management', + 'tags': ['Climatology'] + }, + 'stats': { + 'method': '.*', + 'route': '/malleefowl/.*' + }, + 'redirect': { + 'doc': 'https://malleefowl.readthedocs.io/en/latest/', + 'releasenotes': 'https://github.com/Ouranosinc/malleefowl/blob/master/CHANGES.rst', + 'support': 'https://github.com/Ouranosinc/malleefowl/issues', + 'source': 'https://github.com/Ouranosinc/malleefowl', + 'tryme': 'https://${PAVICS_FQDN_PUBLIC}/malleefowl/', + 'licence': '', + 'provenance': '' + }, + 'monitoring': { + "Malleefowl": { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://${PAVICS_FQDN}:8091/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } + } } -} +} \ No newline at end of file diff --git a/birdhouse/config/malleefowl/default.env b/birdhouse/config/malleefowl/default.env index 5138e6b9f..bc700764d 100644 --- a/birdhouse/config/malleefowl/default.env +++ b/birdhouse/config/malleefowl/default.env @@ -1,6 +1,5 @@ -# When canarie-api is monitoring twitcher, at least one WPS service protected by twitcher -# needs to be available. This sets this component as the service used to monitor twitcher. -export WPS_FOR_TWITCHER_MONITORING=${WPS_FOR_TWITCHER_MONITORING:-malleefowl} + +export MALLEEFOWL_VERSION="pavics-0.3.5" # This tells the postgres service to create a database on startup export POSTGRES_DATABASES_TO_CREATE=" @@ -20,3 +19,8 @@ COMPONENT_DEPENDENCIES=" ./config/wps_outputs-volume ./config/data-volume " + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$MALLEEFOWL_VERSION +" diff --git a/birdhouse/config/malleefowl/docker-compose-extra.yml b/birdhouse/config/malleefowl/docker-compose-extra.yml index 5903cd677..b06114350 100644 --- a/birdhouse/config/malleefowl/docker-compose-extra.yml +++ b/birdhouse/config/malleefowl/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: malleefowl: - image: pavics/malleefowl:pavics-0.3.5 + image: pavics/malleefowl:${MALLEEFOWL_VERSION} container_name: malleefowl environment: HOSTNAME: $HOSTNAME diff --git a/birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py.template index b04e6ff77..1f7330255 100644 --- a/birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/phoenix/config/canarie-api/canarie_api_monitoring.py.template @@ -1,5 +1,34 @@ -SERVICES['node']['monitoring']['Phoenix'] = { - 'request': { - 'url': 'https://${PAVICS_FQDN}:8443/' +SERVICES['Phoenix'] = { + 'info': { + 'name': 'Phoenix', + 'synopsis': 'Legacy authentication. See Magpie/Twitcher instead.', + 'version': "${PHOENIX_VERSION}", + 'institution': 'Ouranos', + 'releaseTime': get_release_time_from_repo_tag("docker", "${PHOENIX_DOCKER}", "${PHOENIX_VERSION}"), + 'researchSubject': 'Authentication', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Authentication', + 'tags': ['Authentication', 'Legacy'] + }, + 'stats': { + 'method': '.*', + 'route': '${TWITCHER_PROTECTED_PATH}/geoserver/web/.*' # FIXME: original value doesn't make sense + }, + 'redirect': { + 'doc': 'http://pyramid-phoenix.readthedocs.io/en/latest/index.html', + 'releasenotes': 'https://github.com/ouranosinc/pyramid-phoenix/CHANGES.rst', + 'support': 'https://github.com/ouranosinc/pyramid-phoenix/issues', + 'source': 'https://github.com/ouranosinc/pyramid-phoenix', + 'tryme': 'https://${PAVICS_FQDN}:8443/', + 'licence': 'https://github.com/ouranosinc/pyramid-phoenix/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + "monitoring": { + "Phoenix": { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'https://${PAVICS_FQDN}:8443/' + } + } } -} +} \ No newline at end of file diff --git a/birdhouse/config/phoenix/default.env b/birdhouse/config/phoenix/default.env index dc97d59b2..abda35660 100644 --- a/birdhouse/config/phoenix/default.env +++ b/birdhouse/config/phoenix/default.env @@ -1,5 +1,8 @@ # All env in this default.env can be overridden by env.local. +export PHOENIX_DOCKER=pavics/pyramid-phoenix +export PHOENIX_VERSION=pavics-0.2.3 + # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here VARS=" $VARS @@ -7,6 +10,12 @@ VARS=" \$PHOENIX_PASSWORD_HASH " +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$PHOENIX_DOCKER + \$PHOENIX_VERSION +" + # add any component that this component requires to run COMPONENT_DEPENDENCIES=" ./config/malleefowl diff --git a/birdhouse/config/phoenix/docker-compose-extra.yml b/birdhouse/config/phoenix/docker-compose-extra.yml index 94d81cfeb..21304f16e 100644 --- a/birdhouse/config/phoenix/docker-compose-extra.yml +++ b/birdhouse/config/phoenix/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: phoenix: - image: pavics/pyramid-phoenix:pavics-0.2.3 + image: ${PHOENIX_DOCKER}:${PHOENIX_VERSION} container_name: phoenix environment: HOSTNAME: $HOSTNAME diff --git a/birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py.template index caec63ebb..060448f2b 100644 --- a/birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/project-api/config/canarie-api/canarie_api_monitoring.py.template @@ -1,6 +1,6 @@ -PLATFORMS['pavics']['monitoring']['Project'] = { +PLATFORMS['server']['monitoring']['Project'] = { 'request': { 'url': 'https://${PAVICS_FQDN_PUBLIC}/project-api/explorer/' } } -PLATFORMS['pavics']['stats']['route'] = '/project-api/.*' +PLATFORMS['server']['stats']['route'] = '/project-api/.*' diff --git a/birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py.template index 13a476a1f..8b22c8441 100644 --- a/birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/raven/config/canarie-api/canarie_api_monitoring.py.template @@ -1,16 +1,16 @@ -SERVICES['node']['monitoring']['Raven'] = { - 'request': { - 'url': 'http://${PAVICS_FQDN}:8096/wps?service=WPS&version=1.0.0&request=GetCapabilities' - } -} +RAVEN_VERSION = "${RAVEN_VERSION}" +RAVEN_RELEASE = get_release_time_from_repo_tag("github", "Ouranosinc/raven", RAVEN_VERSION) + SERVICES['raven'] = { 'info': { 'name': 'Hydrological modeling and analysis', - 'synopsis': ('A suite of WPS processes to calibrate and run hydrological models, including geographical' - 'information retrieval and processing as well as time series analysis.'), - 'version': '0.11.1', + 'synopsis': ( + 'A suite of WPS processes to calibrate and run hydrological models, ' + 'including geographical information retrieval and processing as well as time series analysis.' + ), + 'version': RAVEN_VERSION, 'institution': 'Ouranos', - 'releaseTime': '2021-02-01T00:00:00Z', + 'releaseTime': RAVEN_RELEASE, 'researchSubject': 'Hydrology', 'supportEmail': '${SUPPORT_EMAIL}', 'category': 'Data Manipulation', @@ -32,7 +32,8 @@ SERVICES['raven'] = { 'monitoring': { 'Raven': { 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&version=1.0.0&request=GetCapabilities' + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://${PAVICS_FQDN}:8096/wps?service=WPS&version=1.0.0&request=GetCapabilities' } }, } diff --git a/birdhouse/config/raven/default.env b/birdhouse/config/raven/default.env index 6f88b24d5..8a6cff8d1 100644 --- a/birdhouse/config/raven/default.env +++ b/birdhouse/config/raven/default.env @@ -7,9 +7,7 @@ # Ex: RAVEN_GEO_URL="https://${PAVICS_FQDN}/geoserver/" export RAVEN_GEO_URL="https://pavics.ouranos.ca/geoserver/" -# When canarie-api is monitoring twitcher, at least one WPS service protected by twitcher -# needs to be available. This sets this component as the service used to monitor twitcher. -export WPS_FOR_TWITCHER_MONITORING=${WPS_FOR_TWITCHER_MONITORING:-raven} +export RAVEN_VERSION=0.14.2 # This tells the postgres service to create a database on startup export POSTGRES_DATABASES_TO_CREATE=" @@ -28,3 +26,8 @@ COMPONENT_DEPENDENCIES=" ./config/postgres ./config/wps_outputs-volume " + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$RAVEN_VERSION +" diff --git a/birdhouse/config/raven/docker-compose-extra.yml b/birdhouse/config/raven/docker-compose-extra.yml index 72ed872de..e58f4737a 100644 --- a/birdhouse/config/raven/docker-compose-extra.yml +++ b/birdhouse/config/raven/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: raven: - image: pavics/raven:0.14.2 + image: pavics/raven:${RAVEN_VERSION} container_name: raven ports: - "8096:9099" diff --git a/birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py.template index 5d73600be..557a399f3 100644 --- a/birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/solr/config/canarie-api/canarie_api_monitoring.py.template @@ -1,8 +1,39 @@ -SERVICES['node']['monitoring']['Solr'] = { - 'request': { - 'url': 'http://${PAVICS_FQDN}:8983/solr/birdhouse/select?q=CMIP5&fq=model:MPI-ESM-MR&fq=experiment:rcp45&fq=variable:tasmax&fq=institute:MPI-M&fq=frequency:mon&wt=json' +SERVICES['Solr'] = { + 'info': { + 'name': 'Solr', + 'synopsis': ( + 'SOLR is a search platform part of the Apache Lucene project. ' + 'It is used in this project for its faceted search capability. ' + 'Search queries are relayed from the UI or WPS processes to the SOLR database, ' + 'which returns a json file with the links to matching files.' + ), + 'version': "${SOLR_VERSION}", + 'institution': 'Ouranos', + 'releaseTime': get_release_time_from_repo_tag("docker", "${SOLR_DOCKER}", "${SOLR_VERSION}"), + 'researchSubject': 'Climatology', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Data Manipulation', + 'tags': ['Indexation', 'Search'] }, - 'response': { - 'text': '.*catalog_url\":\".+/testdata/flyingpigeon/cmip5.*/catalog.xml.*' + 'stats': { + 'method': '.*', + 'route': '/solr/.*' + }, + 'redirect': { + 'doc': 'https://ouranosinc.github.io/pavics-sdi/arch/backend.html#indexation', + 'releasenotes': 'https://github.com/Ouranosinc/PAVICS/tags', + 'support': 'https://github.com/Ouranosinc/PAVICS/issues', + 'source': 'https://github.com/Ouranosinc/PAVICS/tree/master/birdhouse/docker/solr', + 'tryme': 'http://${PAVICS_FQDN}:8983/solr/', + 'licence': 'https://github.com/bird-house/finch/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/arch/backend.html#indexation' + }, + "monitoring": { + 'Solr': { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://${PAVICS_FQDN}:8983/solr/birdhouse/select' + } + }, } } diff --git a/birdhouse/config/solr/default.env b/birdhouse/config/solr/default.env new file mode 100644 index 000000000..72291241d --- /dev/null +++ b/birdhouse/config/solr/default.env @@ -0,0 +1,8 @@ +export SOLR_DOCKER=pavics/solr +export SOLR_VERSION=5.2.1 + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$SOLR_DOCKER + \$SOLR_VERSION +" diff --git a/birdhouse/config/solr/docker-compose-extra.yml b/birdhouse/config/solr/docker-compose-extra.yml index df782dfc6..d61c6e5cb 100644 --- a/birdhouse/config/solr/docker-compose-extra.yml +++ b/birdhouse/config/solr/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: solr: - image: pavics/solr:5.2.1 + image: ${SOLR_DOCKER}:${SOLR_VERSION} container_name: solr ports: - "8983:8983" diff --git a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template index 41aa65390..2cdac21fc 100644 --- a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template @@ -1,13 +1,10 @@ -SERVICES['node']['monitoring'].update({'Thredds': { - 'request': { - 'url': 'http://${PAVICS_FQDN}:8083${TWITCHER_PROTECTED_PATH}/thredds/catalog.html' - } - }}) - SERVICES['renderer'] = { 'info': { 'name': 'High-resolution spatial gridded data renderer', - 'synopsis': 'This service renders gridded data on the server and sends images to the client for display within mapping applications using Open Geospatial Consortium (OGC) Web Mappping Service (WMS) standard.', + 'synopsis': ( + 'This service renders gridded data on the server and sends images to the client for display within ' + 'mapping applications using Open Geospatial Consortium (OGC) Web Mappping Service (WMS) standard.' + ), 'version': '4.6.15', 'institution': 'Unidata', 'releaseTime': '2020-06-16T00:00:00Z', @@ -38,4 +35,39 @@ SERVICES['renderer'] = { } } +SERVICES['Thredds'] = { + 'info': { + 'name': 'Thredds', + 'synopsis': 'Climate Data Catalog and Format Renderers', + 'version': "${THREDDS_VERSION}", + 'releaseTime': get_release_time_from_repo_tag("docker", "${THREDDS_DOCKER}", "${THREDDS_VERSION}"), + 'institution': 'Ouranos', + 'researchSubject': 'Catalog', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Catalog', + 'tags': ['Catalog', 'Climate Data'] + }, + 'stats': { + 'method': '.*', + 'route': '${TWITCHER_PROTECTED_PATH}/thredds/.*' + }, + 'redirect': { + 'doc': 'https://www.unidata.ucar.edu/software/tds/', + 'releasenotes': 'https://docs.unidata.ucar.edu/tds/current/userguide/upgrade.html', + 'support': 'https://www.unidata.ucar.edu/software/tds/#help', + 'source': 'https://github.com/Unidata/tds', + 'tryme': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/', + 'licence': 'https://github.com/Unidata/tds/blob/main/LICENSE', + 'provenance': 'https://downloads.unidata.ucar.edu/tds/' + }, + "monitoring": { + "Thredds": { + 'request': { + # FIXME: remove port by design (https://github.com/bird-house/birdhouse-deploy/issues/222) + 'url': 'http://${PAVICS_FQDN}:8083${TWITCHER_PROTECTED_PATH}/thredds/catalog.html' + } + } + } +} + CANARIE_STATS_ROUTES.append('thredds') diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index 30a950818..6e8e6de35 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -1,8 +1,9 @@ # All env in this default.env can be overridden by env.local. # thredds-docker >= 4.6.18 or >= 5.2 strongly recommended to avoid Log4J CVE-2021-44228. -export THREDDS_IMAGE="pavics/thredds-docker:4.6.18-unidata-2022-01" - +export THREDDS_VERSION=4.6.18-unidata-2022-01 +export THREDDS_DOCKER=pavics/thredds-docker +export THREDDS_IMAGE="${THREDDS_DOCKER}:${THREDDS_VERSION}" export THREDDS_ORGANIZATION="Birdhouse" # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here @@ -15,4 +16,6 @@ OPTIONAL_VARS=" $OPTIONAL_VARS \$THREDDS_ORGANIZATION \$TWITCHER_PROTECTED_PATH + \$THREDDS_DOCKER + \$THREDDS_VERSION " diff --git a/birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py.template index 7d4b12c66..2488a57ca 100644 --- a/birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/twitcher/config/canarie-api/canarie_api_monitoring.py.template @@ -1,9 +1,56 @@ -# We can only monitor twitcher if there is an endpoint that it is protecting that we can try to access -# If there is at least one other service that provides a route protected by twitcher, monitor that route; -# otherwise do nothing. -if '${WPS_FOR_TWITCHER_MONITORING}': - SERVICES['node']['monitoring'].update({'Twitcher': { - 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/${WPS_FOR_TWITCHER_MONITORING}?service=WPS&version=1.0.0&request=GetCapabilities' + +def get_twitcher_magpie_adapter_version() -> str: + dockerfile_url = "https://raw.githubusercontent.com/Ouranosinc/Magpie/${MAGPIE_VERSION}/Dockerfile.adapter" + resp = SESSION.get(dockerfile_url) + if resp.status_code != 200: + raise ValueError( + f"HTTP status: {resp.status_code} from [{dockerfile_url}] with detail:\n{resp.text!s}" + ) + tag_line = [line for line in resp.text.splitlines() if line.startswith("FROM birdhouse/twitcher")] + if not tag_line: + raise ValueError(f"Could not locate Twitcher version from [{dockerfile_url}]") + tag_version = tag_line[0].split(":v", 1)[-1].strip() + if not tag_version: + raise ValueError(f"Could not locate Twitcher version from [{dockerfile_url}]") + return tag_version + + +TWITCHER_VERSION = get_twitcher_magpie_adapter_version() +TWITCHER_RELEASE = get_release_time_from_repo_tag("github", "bird-house/twitcher", TWITCHER_VERSION) +SERVICES['Twitcher'] = { + 'info': { + 'name': 'Twitcher', + 'synopsis': ( + 'Twitcher is a security proxy for OWS services like Web Processing Services (WPS). ' + 'The proxy service uses OAuth2 access tokens to protect the OWS service access using ' + 'Magpie permissions.' + ), + 'version': TWITCHER_VERSION, + 'institution': 'Ouranos', + 'releaseTime': TWITCHER_RELEASE, + 'researchSubject': 'Security', + 'supportEmail': '${SUPPORT_EMAIL}', + 'category': 'Security', + 'tags': ['Security', 'Proxy', 'Access', 'Policy Enforcement Point'] + }, + 'stats': { + 'method': '.*', + 'route': "/twitcher/(?!ows).*" + }, + 'redirect': { + 'doc': 'https://twitcher.readthedocs.io/', + 'releasenotes': 'https://github.com/bird-house/twitcher/blob/master/CHANGES.rst', + 'support': 'https://github.com/bird-house/twitcher/issues', + 'source': 'https://github.com/bird-house/twitcher', + 'tryme': 'https://${PAVICS_FQDN_PUBLIC}/twitcher/', + 'licence': 'https://github.com/bird-house/twitcher/blob/master/LICENSE.txt', + 'provenance': 'https://ouranosinc.github.io/pavics-sdi/provenance/index.html' + }, + "monitoring": { + "Twitcher": { + 'request': { + 'url': 'https://${PAVICS_FQDN_PUBLIC}/twitcher/' + } } - }}) + } +} diff --git a/birdhouse/config/twitcher/default.env b/birdhouse/config/twitcher/default.env index b71d5d91c..55fe42be0 100644 --- a/birdhouse/config/twitcher/default.env +++ b/birdhouse/config/twitcher/default.env @@ -1,5 +1,6 @@ # All env in this default.env can be overridden by env.local. +# Note: Twitcher version is determined by the associated MAGPIE_VERSION. export TWITCHER_LOG_LEVEL=INFO export VERIFY_SSL="true" @@ -13,7 +14,6 @@ VARS=" OPTIONAL_VARS=" $OPTIONAL_VARS \$TWITCHER_LOG_LEVEL - \$WPS_FOR_TWITCHER_MONITORING " # add any component that this component requires to run diff --git a/birdhouse/default.env b/birdhouse/default.env index 5620339ec..e67028d34 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -26,6 +26,16 @@ export DELAYED_EVAL=" PAVICS_FQDN_PUBLIC " + +export SERVER_NAME=PAVICS +export SERVER_DESCRIPTION=" +The PAVICS (Power Analytics for Visualization of Climate Science) platform is a collection of +climate analysis services served through Open Geospatial Consortium (OGC) protocols. +These services include data access, processing and visualization. Both data and algorithms +can be accessed either programmatically, through OGC-compliant clients such as QGIS or ArcGIS, +or a custom web interface. +" + export DEFAULT_CONF_DIRS=' ./config/proxy ./config/canarie-api diff --git a/birdhouse/optional-components/all-public-access/config/ncwms2/docker-compose-extra.yml b/birdhouse/optional-components/all-public-access/config/ncwms2/docker-compose-extra.yml new file mode 100644 index 000000000..5c9ec6155 --- /dev/null +++ b/birdhouse/optional-components/all-public-access/config/ncwms2/docker-compose-extra.yml @@ -0,0 +1,5 @@ +version: "3.4" +services: + magpie: + volumes: + - ./optional-components/all-public-access/config/ncwms2/permissions.cfg:${MAGPIE_PERMISSIONS_CONFIG_PATH}/all-public-access-ncwms2-permissions.cfg:ro diff --git a/birdhouse/optional-components/all-public-access/config/ncwms2/permissions.cfg b/birdhouse/optional-components/all-public-access/config/ncwms2/permissions.cfg new file mode 100644 index 000000000..a4af5b711 --- /dev/null +++ b/birdhouse/optional-components/all-public-access/config/ncwms2/permissions.cfg @@ -0,0 +1,5 @@ +permissions: + - service: ncWMS2 + permission: getcapabilities + group: anonymous + action: create diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py index beaa267be..25a0c455a 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/a_demo_override_precedence.py @@ -1,10 +1,11 @@ # This file will be overridden by canarie_api_full_monitoring.py.template # because they are loaded in alphabetical sorted order and the last one have -# highest override precedence. -SERVICES['node']['monitoring'].update({ - 'Thredds-public': { - 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}/toto', - }, - }, -}) +# the highest override precedence. + +# SERVICES['node']['monitoring'].update({ +# 'Thredds-public': { +# 'request': { +# 'url': 'https://${PAVICS_FQDN_PUBLIC}/toto', +# }, +# }, +# }) diff --git a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/canarie_api_full_monitoring.py.template b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/canarie_api_full_monitoring.py.template index c7dd29b53..62df02d75 100644 --- a/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/canarie_api_full_monitoring.py.template +++ b/birdhouse/optional-components/canarie-api-full-monitoring/config/canarie-api/canarie_api_full_monitoring.py.template @@ -1,4 +1,6 @@ -SERVICES['node']['monitoring'].update({ +import copy + +_BIRDS = { 'Thredds-public': { 'request': { 'url': 'https://${PAVICS_FQDN_PUBLIC}/thredds/catalog.html', @@ -30,11 +32,33 @@ SERVICES['node']['monitoring'].update({ 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&version=1.0.0&request=GetCapabilities' } }, + # FIXME: remove (https://github.com/bird-house/birdhouse-deploy/issues/214) 'ncWMS2-public': { 'request': { 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0' } }, -}) +} +for bird_public, bird_public_cfg in _BIRDS.items(): + bird_private = bird_public.split("-public")[0] + if bird_private in SERVICES: + SERVICES[bird_private]["monitoring"][bird_public] = bird_public_cfg + elif "node" in SERVICES: + SERVICES["node"]["monitoring"][bird_public] = bird_public_cfg + + +# See notes in 'components/weaver/docker-compose-extra.yml' +# about the conditional loading of this config in 'proxy' service. +if "Weaver" in SERVICES: + weaver_cfg = copy.deepcopy(SERVICES["Weaver"]["monitoring"]["Weaver"]) + weaver_cfg["request"]["url"] = "https://${PAVICS_FQDN_PUBLIC}/weaver/" + SERVICES["Weaver"]["monitoring"]["Weaver-public"] = weaver_cfg + +# See notes in 'components/cowbird/docker-compose-extra.yml' +# about the conditional loading of this config in 'proxy' service. +if "Cowbird" in SERVICES: + cowbird_cfg = copy.deepcopy(SERVICES["Cowbird"]["monitoring"]["Cowbird"]) + cowbird_cfg["request"]["url"] = "https://${PAVICS_FQDN_PUBLIC}/cowbird/" + SERVICES["Cowbird"]["monitoring"]["Cowbird-public"] = cowbird_cfg # vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/optional-components/emu/default.env b/birdhouse/optional-components/emu/default.env index a99795288..bac9af103 100644 --- a/birdhouse/optional-components/emu/default.env +++ b/birdhouse/optional-components/emu/default.env @@ -15,14 +15,6 @@ export EMU_NAME="emu" # set to 'testwps_outputs' when used together with testthredds component export EMU_WPS_OUTPUTS_VOL="wps_outputs" -# When canarie-api is monitoring twitcher, at least one WPS service protected by twitcher -# needs to be available. This sets this component as the service used to monitor twitcher. -export WPS_FOR_TWITCHER_MONITORING='${EMU_NAME}' - -export DELAYED_EVAL=" - $DELAYED_EVAL - WPS_FOR_TWITCHER_MONITORING -" # add vars only needed to be substituted in templates OPTIONAL_VARS=" diff --git a/birdhouse/optional-components/generic_bird/default.env b/birdhouse/optional-components/generic_bird/default.env index 41c963078..f170e08e7 100644 --- a/birdhouse/optional-components/generic_bird/default.env +++ b/birdhouse/optional-components/generic_bird/default.env @@ -11,15 +11,6 @@ export GENERIC_BIRD_INTERNAL_PORT="5000" export GENERIC_BIRD_NAME="generic_bird" export GENERIC_BIRD_POSTGRES_IMAGE="postgres:10.12" -# When canarie-api is monitoring twitcher, at least one WPS service protected by twitcher -# needs to be available. This sets this component as the service used to monitor twitcher. -export WPS_FOR_TWITCHER_MONITORING='${GENERIC_BIRD_NAME}' - -export DELAYED_EVAL=" - $DELAYED_EVAL - WPS_FOR_TWITCHER_MONITORING -" - # add vars only needed to be substituted in templates OPTIONAL_VARS=" diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index 3e4408bba..ed37f48e8 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -31,6 +31,8 @@ VARS=' OPTIONAL_VARS=' $PAVICS_FQDN_PUBLIC $EXTRA_PYWPS_CONFIG + $SERVER_NAME + $SERVER_DESCRIPTION ' # we switch to the real directory of the script, so it still works when used from $PATH diff --git a/docs/source/conf.py b/docs/source/conf.py index af6c5cf75..df461b6b2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '1.25.7' +version = '1.26.0' # The full version, including alpha/beta/rc tags. -release = '1.25.7' +release = '1.26.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/tests/test_read_configs_include.py b/tests/test_read_configs_include.py index 3fe5e2afe..8ce478875 100644 --- a/tests/test_read_configs_include.py +++ b/tests/test_read_configs_include.py @@ -171,6 +171,7 @@ class TestCreateComposeConfList: "./config/malleefowl/config/data-volume/docker-compose-extra.yml", "./config/malleefowl/config/wps_outputs-volume/docker-compose-extra.yml", "./config/flyingpigeon/docker-compose-extra.yml", + "./config/flyingpigeon/config/canarie-api/docker-compose-extra.yml", "./config/flyingpigeon/config/wps_outputs-volume/docker-compose-extra.yml", "./config/catalog/docker-compose-extra.yml", "./config/catalog/config/canarie-api/docker-compose-extra.yml", @@ -187,6 +188,7 @@ class TestCreateComposeConfList: "./config/raven/config/canarie-api/docker-compose-extra.yml", "./config/raven/config/wps_outputs-volume/docker-compose-extra.yml", "./config/hummingbird/docker-compose-extra.yml", + "./config/hummingbird/config/canarie-api/docker-compose-extra.yml", "./config/hummingbird/config/data-volume/docker-compose-extra.yml", "./config/hummingbird/config/wps_outputs-volume/docker-compose-extra.yml", "./config/thredds/docker-compose-extra.yml",