From 526af44d3477a6547e8b1a65b3e516c053553bd4 Mon Sep 17 00:00:00 2001 From: "Alan B. Christie" <29806285+alanbchristie@users.noreply.github.com> Date: Mon, 19 Feb 2024 18:22:51 +0100 Subject: [PATCH] Prepare for first formal release (#537) * Some changes to cset_upload.py to allow site observation short codes (#527) * stashing * fix: cset_upload.py updated to allow new-style site observation codes NB! this probably still won't work! I suspect the file I was given is broken and I cannot test it further * stashing * Short code prefix and tooltip to backend Target loader now reads short code prefix and tooltip from meta_aligner.yaml. Tooltip is saved to Experiment model. TODO: make tooltip available via API * Prefix tooltip now serverd by api/site_observation * Site observation groups for shortcodes now by experiment * New format to download zip (issue 1326) (#530) * stashing * stashing * feat: download structure fixed TODO: add all the yamls * All yaml files added to download * cset_upload.py: lhs_pdb renamed to ref_pdb * Renamed canon- and conf site tags * Adds support for key-based SSH connections (#534) * Centralised environment variables (#529) * refactor: Restructured settings.py * docs: Minor tweaks * refactor: Move security and infection config to settings * refactor: b/e & f/e/ tags now in settings (also fixed f/e tag value) * refactor: Move Neo4j config to settings * refactor: More variables into settings * refactor: Moved remaining config * docs: Adds configuration guide as comments * docs: Variable prefix now 'stack_' not 'stack_env_' --------- Co-authored-by: Alan Christie * feat: Adds support for private keys on SSH tunnel * fix: Fixes key-based logic --------- Co-authored-by: Alan Christie * build(deps): bump cryptography from 42.0.0 to 42.0.2 (#533) Bumps [cryptography](https://github.com/pyca/cryptography) from 42.0.0 to 42.0.2. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.0...42.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * docs: Updates documentation (#536) Co-authored-by: Alan Christie * build(deps): bump django from 3.2.20 to 3.2.24 (#535) Bumps [django](https://github.com/django/django) from 3.2.20 to 3.2.24. - [Commits](https://github.com/django/django/compare/3.2.20...3.2.24) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: Kalev Takkis Co-authored-by: Warren Thompson Co-authored-by: Alan Christie Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- README.md | 14 +- api/infections.py | 12 +- api/remote_ispyb_connector.py | 54 +- api/security.py | 39 +- build-requirements.txt | 2 +- fragalysis/settings.py | 656 +++++++++++------- fragalysis/views.py | 39 +- network/views.py | 13 +- poetry.lock | 66 +- viewer/cset_upload.py | 27 +- viewer/download_structures.py | 179 ++++- viewer/managers.py | 1 + .../0043_experiment_prefix_tooltip.py | 17 + viewer/models.py | 1 + viewer/sdf_check.py | 8 +- viewer/serializers.py | 5 +- viewer/services.py | 5 +- viewer/squonk2_agent.py | 63 +- viewer/target_loader.py | 42 +- viewer/views.py | 4 +- 20 files changed, 755 insertions(+), 492 deletions(-) create mode 100644 viewer/migrations/0043_experiment_prefix_tooltip.py diff --git a/README.md b/README.md index 0b417e5e..af572d65 100644 --- a/README.md +++ b/README.md @@ -66,13 +66,11 @@ installs/updates new packages to local venv. It's equivalent to running `poetry lock && poetry install`, so if you're not interested in local environment and just want to update the lockfile, you can run just `poetry lock`. - ## Building and running (local) The backend is a Docker container image and can be build and deployed locally using `docker-compose`: - docker-compose build - To run the application (which wil include deployment of the postgres and neo4j databases) run: - @@ -181,6 +179,18 @@ at `/code/logs`. > For local development using the `docker-compose.yml` file you'll find the logs at `./data/logs/backend.log`. +## Configuration (environment variables) +The backend configuration is controlled by a number of environment variables. +Variables are typically defined in the project's `fragalysis/settings.py`, where you +will also find **ALL** the dynamically configured variables (those that can be changed +using *environment variables* in the deployed Pod/Container). + +- Not all variables are dynamic. For example `ALLOWED_HOSTS` is a static variable + that is set in the `settings.py` file and is not intended to be changed at run-time. + +Refer to the documentation in the `settings.py` file to understand the environment +and the style guide for new variables that you need to add. + ## Database migrations The best approach is to spin-up the development backend (locally) using `docker-compose` with the custom *migration* compose file and then shell into Django. diff --git a/api/infections.py b/api/infections.py index c1eb6cab..4143c585 100644 --- a/api/infections.py +++ b/api/infections.py @@ -4,9 +4,10 @@ # Infections are injected into the application via the environment variable # 'INFECTIONS', a comma-separated list of infection names. -import os from typing import Dict, Set +from django.conf import settings + from api.utils import deployment_mode_is_production # The built-in set of infections. @@ -20,9 +21,6 @@ INFECTION_STRUCTURE_DOWNLOAD: 'An error in the DownloadStructures view' } -# What infection have been set? -_INFECTIONS: str = os.environ.get('INFECTIONS', '').lower() - def have_infection(name: str) -> bool: """Returns True if we've been given the named infection. @@ -31,9 +29,11 @@ def have_infection(name: str) -> bool: def _get_infections() -> Set[str]: - if _INFECTIONS == '': + if settings.INFECTIONS == '': return set() infections: set[str] = { - infection for infection in _INFECTIONS.split(',') if infection in _CATALOGUE + infection + for infection in settings.INFECTIONS.split(',') + if infection in _CATALOGUE } return infections diff --git a/api/remote_ispyb_connector.py b/api/remote_ispyb_connector.py index 56fce7dc..398f3473 100644 --- a/api/remote_ispyb_connector.py +++ b/api/remote_ispyb_connector.py @@ -28,6 +28,7 @@ def __init__( remote=False, ssh_user=None, ssh_password=None, + ssh_private_key_filename=None, ssh_host=None, conn_inactivity=360, ): @@ -45,6 +46,7 @@ def __init__( 'ssh_host': ssh_host, 'ssh_user': ssh_user, 'ssh_pass': ssh_password, + 'ssh_pkey': ssh_private_key_filename, 'db_host': host, 'db_port': int(port), 'db_user': user, @@ -53,12 +55,11 @@ def __init__( } self.remote_connect(**creds) logger.debug( - "Started host=%s username=%s local_bind_port=%s", + "Started remote ssh_host=%s ssh_user=%s local_bind_port=%s", ssh_host, ssh_user, self.server.local_bind_port, ) - else: self.connect( user=user, @@ -68,29 +69,60 @@ def __init__( port=port, conn_inactivity=conn_inactivity, ) - logger.debug("Started host=%s user=%s port=%s", host, user, port) + logger.debug("Started direct host=%s user=%s port=%s", host, user, port) def remote_connect( - self, ssh_host, ssh_user, ssh_pass, db_host, db_port, db_user, db_pass, db_name + self, + ssh_host, + ssh_user, + ssh_pass, + ssh_pkey, + db_host, + db_port, + db_user, + db_pass, + db_name, ): sshtunnel.SSH_TIMEOUT = 10.0 sshtunnel.TUNNEL_TIMEOUT = 10.0 sshtunnel.DEFAULT_LOGLEVEL = logging.CRITICAL self.conn_inactivity = int(self.conn_inactivity) - self.server = sshtunnel.SSHTunnelForwarder( - (ssh_host), - ssh_username=ssh_user, - ssh_password=ssh_pass, - remote_bind_address=(db_host, db_port), - ) + if ssh_pkey: + logger.debug( + 'Creating SSHTunnelForwarder (with SSH Key) host=%s user=%s', + ssh_host, + ssh_user, + ) + self.server = sshtunnel.SSHTunnelForwarder( + (ssh_host), + ssh_username=ssh_user, + ssh_pkey=ssh_pkey, + remote_bind_address=(db_host, db_port), + ) + else: + logger.debug( + 'Creating SSHTunnelForwarder (with password) host=%s user=%s', + ssh_host, + ssh_user, + ) + self.server = sshtunnel.SSHTunnelForwarder( + (ssh_host), + ssh_username=ssh_user, + ssh_password=ssh_pass, + remote_bind_address=(db_host, db_port), + ) + logger.debug('Created SSHTunnelForwarder') # stops hanging connections in transport self.server.daemon_forward_servers = True self.server.daemon_transport = True + logger.debug('Starting SSH server...') self.server.start() + logger.debug('Started SSH server') + logger.debug('Connecting to ISPyB (db_user=%s db_name=%s)...', db_user, db_name) self.conn = pymysql.connect( user=db_user, password=db_pass, @@ -100,8 +132,10 @@ def remote_connect( ) if self.conn is not None: + logger.debug('Connected') self.conn.autocommit = True else: + logger.debug('Failed to connect') self.server.stop() raise ISPyBConnectionException self.last_activity_ts = time.time() diff --git a/api/security.py b/api/security.py index eafc31fe..01605352 100644 --- a/api/security.py +++ b/api/security.py @@ -48,40 +48,41 @@ def get_remote_conn() -> Optional[SSHConnector]: - ispyb_credentials: Dict[str, Any] = { - "user": os.environ.get("ISPYB_USER"), - "pw": os.environ.get("ISPYB_PASSWORD"), - "host": os.environ.get("ISPYB_HOST"), - "port": os.environ.get("ISPYB_PORT"), + credentials: Dict[str, Any] = { + "user": settings.ISPYB_USER, + "pw": settings.ISPYB_PASSWORD, + "host": settings.ISPYB_HOST, + "port": settings.ISPYB_PORT, "db": "ispyb", "conn_inactivity": 360, } ssh_credentials: Dict[str, Any] = { - 'ssh_host': os.environ.get("SSH_HOST"), - 'ssh_user': os.environ.get("SSH_USER"), - 'ssh_password': os.environ.get("SSH_PASSWORD"), + 'ssh_host': settings.SSH_HOST, + 'ssh_user': settings.SSH_USER, + 'ssh_password': settings.SSH_PASSWORD, + "ssh_private_key_filename": settings.SSH_PRIVATE_KEY_FILENAME, 'remote': True, } - ispyb_credentials.update(**ssh_credentials) + credentials.update(**ssh_credentials) # Caution: Credentials may not be set in the environment. # Assume the credentials are invalid if there is no host. # If a host is not defined other properties are useless. - if not ispyb_credentials["host"]: + if not credentials["host"]: logger.debug("No ISPyB host - cannot return a connector") return None # Try to get an SSH connection (aware that it might fail) conn: Optional[SSHConnector] = None try: - conn = SSHConnector(**ispyb_credentials) + conn = SSHConnector(**credentials) except Exception: # Log the exception if DEBUG level or lower/finer? - # The following wil not log if the level is set to INFO for example. + # The following will not log if the level is set to INFO for example. if logging.DEBUG >= logger.level: - logger.info("ispyb_credentials=%s", ispyb_credentials) + logger.info("credentials=%s", credentials) logger.exception("Got the following exception creating SSHConnector...") return conn @@ -89,10 +90,10 @@ def get_remote_conn() -> Optional[SSHConnector]: def get_conn() -> Optional[Connector]: credentials: Dict[str, Any] = { - "user": os.environ.get("ISPYB_USER"), - "pw": os.environ.get("ISPYB_PASSWORD"), - "host": os.environ.get("ISPYB_HOST"), - "port": os.environ.get("ISPYB_PORT"), + "user": settings.ISPYB_USER, + "pw": settings.ISPYB_PASSWORD, + "host": settings.ISPYB_HOST, + "port": settings.ISPYB_PORT, "db": "ispyb", "conn_inactivity": 360, } @@ -108,7 +109,7 @@ def get_conn() -> Optional[Connector]: conn = Connector(**credentials) except Exception: # Log the exception if DEBUG level or lower/finer? - # The following wil not log if the level is set to INFO for example. + # The following will not log if the level is set to INFO for example. if logging.DEBUG >= logger.level: logger.info("credentials=%s", credentials) logger.exception("Got the following exception creating Connector...") @@ -349,7 +350,7 @@ def get_proposals_for_user(self, user, restrict_to_membership=False): assert user proposals = set() - ispyb_user = os.environ.get("ISPYB_USER") + ispyb_user = settings.ISPYB_USER logger.debug( "ispyb_user=%s restrict_to_membership=%s", ispyb_user, diff --git a/build-requirements.txt b/build-requirements.txt index 3a824035..22008213 100644 --- a/build-requirements.txt +++ b/build-requirements.txt @@ -8,7 +8,7 @@ pre-commit == 3.5.0 poetry == 1.7.1 # Matching main requirements... -Django==3.2.20 +Django==3.2.24 # Others httpie == 3.2.1 diff --git a/fragalysis/settings.py b/fragalysis/settings.py index 93885b01..c7b47cc4 100644 --- a/fragalysis/settings.py +++ b/fragalysis/settings.py @@ -1,18 +1,73 @@ -""" -Django settings for fragalysis project. +"""Django settings for the fragalysis 'backend'""" -Generated by 'django-admin startproject' using Django 1.11.6. - -For more information on this file, see -https://docs.djangoproject.com/en/1.11/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.11/ref/settings/ -""" +# This standard Django module is used to provide the dynamic configuration of +# the backend logic. As well as providing vital django-related configuration +# it is also the source of the numerous fragalysis-specific environment variables +# that control the stack's configuration (behaviour). +# +# Not all settings are configured by environment variable. Some are hard-coded +# and you'll need to edit their values here. For example `ALLOWED_HOSTS` +# is a static variable that is not intended to be changed at run-time. +# +# Those that are configurable at run-time should be obvious +# (i.e. they'll use "os.environ.get()" to obtain their value) alternative run-time value. +# +# You will find the django-related configuration at the top of the file +# (under DJANGO SETTINGS) and the fragalysis-specific configuration at the bottom of +# the file (under FRAGALYSIS SETTINGS). +# +# Guidance for variables: - +# +# 1. Everything *MUST* have a default value, this file should not raise an exception +# if a value cannot be found in the environment, that's the role of the +# application code. +# +# 2. The constant used to hold the environment variable *SHOULD* match the +# environment variable's name. i.e. the "DEPLOYMENT_MODE" environment variable's +# value *SHOULD* be found in 'settings.DEPLOYMENT_MODE' variable. +# +# 3. In the FRAGALYSIS section, document the variable's purpose and the values +# it can take in the comments. If there are dependencies or "gotchas" +# (i.e. changing its value after deployment) then these should be documented. +# +# Providing run-time values for variables: - +# +# The environment variable values are set using either a 'docker-compose' file +# (when used for local development) or, more typically, via an "Ansible variable" +# provided by the "Ansible playbook" that's responsible for deploying the stack. +# +# Many (not all) of the environment variables are made available +# for deployment using an Ansible playbook variable, explained below. +# +# 1. Ansible variables are lower-case and use "snake case". +# +# 2. Ansible variables that map directly to environment variables in this file +# use the same name as the environment variable and are prefixed with +# "stack_". For example the "DEPLOYMENT_MODE" environment variable +# can be set using the "stack_deployment_mode" variable. +# +# 3. Variables are declared using the 'EXTRA VARIABLES' section of the corresponding +# AWX "Job Template". +# +# IMPORTANTLY: For a description of an environment variable (setting) and its value +# you *MUST* consult the comments in this file ("settings.py"), and *NOT* +# the Ansible playbook. "settings.py" is the primary authority for the +# configuration of the Fragalysis Stack. +# +# Ansible variables are declared in "roles/fragalysis-stack/defaults/main.yaml" +# or "roles/fragalysis-stack/vars/main.yaml" of the playbook repository +# https://github.com/xchem/fragalysis-stack-kubernetes +# +# For more information on "settings.py", see +# https://docs.djangoproject.com/en/3.2/topics/settings/ +# +# For the full list of Django-related settings and their values, see +# https://docs.djangoproject.com/en/3.2/ref/settings/ import os import sys from datetime import timedelta +from typing import List import sentry_sdk from sentry_sdk.integrations.celery import CeleryIntegration @@ -20,88 +75,52 @@ from sentry_sdk.integrations.excepthook import ExcepthookIntegration from sentry_sdk.integrations.redis import RedisIntegration -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False -if os.environ.get("DEBUG_FRAGALYSIS") == True: - DEBUG = True +# -------------------------------------------------------------------------------------- +# DJANGO SETTINGS +# -------------------------------------------------------------------------------------- -# These flags are used in the upload_tset form as follows. -# Proposal Supported | Proposal Required | Proposal / View fields -# Y | Y | Shown / Required -# Y | N | Shown / Optional -# N | N | Not Shown -PROPOSAL_SUPPORTED = True -PROPOSAL_REQUIRED = True +ALLOWED_HOSTS = ["*"] # AnonymousUser should be the first record inserted into the auth_user table. ANONYMOUS_USER = 1 -# This is set on AWX when the fragalysis-stack is rebuilt. -SENTRY_DNS = os.environ.get("FRAGALYSIS_BACKEND_SENTRY_DNS") -if SENTRY_DNS: - # By default only call sentry in staging/production - sentry_sdk.init( - dsn=SENTRY_DNS, - integrations=[ - DjangoIntegration(), - CeleryIntegration(), - RedisIntegration(), - ExcepthookIntegration(always_run=True), - ], - # If you wish to associate users to errors (assuming you are using - # django.contrib.auth) you may enable sending PII data. - send_default_pii=True, - ) - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.environ.get( - "WEB_DJANGO_SECRET_KEY", "8flmz)c9i!o&f1-moi5-p&9ak4r9=ck$3!0y1@%34p^(6i*^_9" +AUTHENTICATION_BACKENDS = ( + "django.contrib.auth.backends.ModelBackend", + "fragalysis.auth.KeycloakOIDCAuthenticationBackend", + "guardian.backends.ObjectPermissionBackend", ) -USE_X_FORWARDED_HOST = True -SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") - -ALLOWED_HOSTS = ["*"] - -DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" +# Password validation +# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators -# DATA_UPLOAD_MAX_MEMORY_SIZE = 26214400 # 25 MB +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" + }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, +] -REST_FRAMEWORK = { - "DEFAULT_FILTER_BACKENDS": ("django_filters.rest_framework.DjangoFilterBackend",), - "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", - "PAGE_SIZE": 5000, - "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.QueryParameterVersioning", - 'DEFAULT_AUTHENTICATION_CLASSES': [ - 'rest_framework.authentication.SessionAuthentication', - 'mozilla_django_oidc.contrib.drf.OIDCAuthentication', - 'rest_framework.authentication.BasicAuthentication', - ], -} +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # CELERY STUFF -CELERY_ACCEPT_CONTENT = ['application/json'] +CELERY_ACCEPT_CONTENT = ["application/json"] CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True -CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://redis:6379/') -CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://redis:6379/0') +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://redis:6379/") +CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", "redis://redis:6379/0") CELERY_RESULT_BACKEND_ALWAYS_RETRY = True CELERY_RESULT_EXPIRES = timedelta(days=15) CELERY_TASK_ALWAYS_EAGER = os.environ.get( - 'CELERY_TASK_ALWAYS_EAGER', 'False' -).lower() in ['true', 'yes'] + "CELERY_TASK_ALWAYS_EAGER", "False" +).lower() in ["true", "yes"] CELERY_WORKER_HIJACK_ROOT_LOGGER = False -# This can be injected as an ENV var -NEOMODEL_NEO4J_BOLT_URL = os.environ.get( - "NEO4J_BOLT_URL", "bolt://neo4j:test@neo4j:7687" -) +# SECURITY WARNING: don't run with DUBUG turned on in production! +DEBUG = os.environ.get("DEBUG_FRAGALYSIS") == "True" + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" # Application definition INSTALLED_APPS = [ @@ -136,6 +155,17 @@ "simple_history", ] +LANGUAGE_CODE = "en-us" + +# Swagger logging / logout +LOGIN_URL = "/accounts/login/" +LOGOUT_URL = "/accounts/logout/" +# LOGIN_REDIRECT_URL = "" +LOGIN_REDIRECT_URL = "/viewer/react/landing" +# LOGOUT_REDIRECT_URL = "" +LOGOUT_REDIRECT_URL = "/viewer/react/landing" + MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", @@ -147,25 +177,88 @@ "mozilla_django_oidc.middleware.SessionRefresh", ] -AUTHENTICATION_BACKENDS = ( - "django.contrib.auth.backends.ModelBackend", - "fragalysis.auth.KeycloakOIDCAuthenticationBackend", - "guardian.backends.ObjectPermissionBackend", +PROJECT_ROOT = os.path.abspath(os.path.join(BASE_DIR, "..")) + +REST_FRAMEWORK = { + "DEFAULT_FILTER_BACKENDS": ("django_filters.rest_framework.DjangoFilterBackend",), + "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", + "PAGE_SIZE": 5000, + "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.QueryParameterVersioning", + "DEFAULT_AUTHENTICATION_CLASSES": [ + "rest_framework.authentication.SessionAuthentication", + "mozilla_django_oidc.contrib.drf.OIDCAuthentication", + "rest_framework.authentication.BasicAuthentication", + ], +} + +ROOT_URLCONF = "fragalysis.urls" + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get( + "WEB_DJANGO_SECRET_KEY", "8flmz)c9i!o&f1-moi5-p&9ak4r9=ck$3!0y1@%34p^(6i*^_9" ) -STATICFILES_DIRS = [os.path.join(BASE_DIR, "fragalysis", "../viewer/static")] +if SENTRY_DNS := os.environ.get("FRAGALYSIS_BACKEND_SENTRY_DNS"): + # By default only call sentry in staging/production + sentry_sdk.init( + dsn=SENTRY_DNS, + integrations=[ + DjangoIntegration(), + CeleryIntegration(), + RedisIntegration(), + ExcepthookIntegration(always_run=True), + ], + # If you wish to associate users to errors (assuming you are using + # django.contrib.auth) you may enable sending PII data. + send_default_pii=True, + ) +STATIC_ROOT = os.path.join(PROJECT_ROOT, "static") +STATICFILES_DIRS = [os.path.join(BASE_DIR, "fragalysis", "../viewer/static")] STATICFILES_FINDERS = ( "django.contrib.staticfiles.finders.FileSystemFinder", "django.contrib.staticfiles.finders.AppDirectoriesFinder", ) -# mozilla_django_oidc - from documentation: https://mozilla-django-oidc.readthedocs.io/en/stable/ -# Before you can configure your application, you need to set up a client with an OpenID Connect provider (OP). -# You’ll need to set up a different client for every environment you have for your site. For example, -# if your site has a -dev, -stage, and -prod environments, each of those has a different hostname and thus you +USE_X_FORWARDED_HOST = True +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + +# A list of identifiers of messages generated by the system check framework +# that we wish to permanently acknowledge and ignore. +# Silenced checks will not be output to the console. +# +# fields.W342 Is issued for the xchem-db package. +# The hint is "ForeignKey(unique=True) is usually better served by a OneToOneField." +SILENCED_SYSTEM_CHECKS = [ + "fields.W342", +] + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ] + }, + } +] + +TIME_ZONE = "UTC" + +# mozilla_django_oidc. +# See: https://mozilla-django-oidc.readthedocs.io/en/stable/ +# Before you can configure your application, you need to set up a client with +# an OpenID Connect provider (OP). You’ll need to set up a different client for +# every environment you have for your site. For example, if your site has a -dev, +# -stage, and -prod environments, each of those has a different hostname and thus you # need to set up a separate client for each one. -# you need to provide your OpenID Connect provider (OP) the callback url for your site. +# You need to provide your OpenID Connect provider (OP) the callback url for your site. # The URL path for the callback url is /oidc/callback/. # # Here are examples of callback urls: @@ -179,14 +272,18 @@ # a client id (OIDC_RP_CLIENT_ID) # a client secret (OIDC_RP_CLIENT_SECRET) -# Keycloak mozilla_django_oidc - Settings -# from keyclaok (openid provider = OP) - NB these should be environment variables - not checked in +# Keycloak mozilla_django_oidc settings (openid provider = OP). +# These should be environment variables - not checked in OIDC_RP_CLIENT_ID = os.environ.get("OIDC_RP_CLIENT_ID", "fragalysis-local") -OIDC_RP_CLIENT_SECRET = os.environ.get('OIDC_RP_CLIENT_SECRET') +OIDC_RP_CLIENT_SECRET = os.environ.get("OIDC_RP_CLIENT_SECRET") OIDC_KEYCLOAK_REALM = os.environ.get( "OIDC_KEYCLOAK_REALM", "https://keycloak.xchem-dev.diamond.ac.uk/auth/realms/xchem" ) +# Squonk2 Account Server and Data Manager Client IDs +OIDC_AS_CLIENT_ID: str = os.environ.get("OIDC_AS_CLIENT_ID", "") +OIDC_DM_CLIENT_ID: str = os.environ.get("OIDC_DM_CLIENT_ID", "") + # OIDC_OP_AUTHORIZATION_ENDPOINT = "" OIDC_OP_AUTHORIZATION_ENDPOINT = os.path.join( OIDC_KEYCLOAK_REALM, "protocol/openid-connect/auth" @@ -199,11 +296,13 @@ OIDC_OP_USER_ENDPOINT = os.path.join( OIDC_KEYCLOAK_REALM, "protocol/openid-connect/userinfo" ) -# OIDC_OP_JWKS_ENDPOINT = "" - This is required when using RS256. +# OIDC_OP_JWKS_ENDPOINT = "" +# This is required when using RS256. OIDC_OP_JWKS_ENDPOINT = os.path.join( OIDC_KEYCLOAK_REALM, "protocol/openid-connect/certs" ) -# OIDC_OP_LOGOUT_ENDPOINT = "" - This is required when using RS256. +# OIDC_OP_LOGOUT_ENDPOINT = "" +# This is required when using RS256. OIDC_OP_LOGOUT_ENDPOINT = os.path.join( OIDC_KEYCLOAK_REALM, "protocol/openid-connect/logout" ) @@ -212,76 +311,23 @@ # If desired, this should be set to "fragalysis.views.keycloak_logout" OIDC_OP_LOGOUT_URL_METHOD = os.environ.get("OIDC_OP_LOGOUT_URL_METHOD") -# LOGIN_REDIRECT_URL = "" -LOGIN_REDIRECT_URL = "/viewer/react/landing" -# LOGOUT_REDIRECT_URL = "" -LOGOUT_REDIRECT_URL = "/viewer/react/landing" - # After much trial and error -# Using RS256 + JWKS Endpoint seems to work with no value for OIDC_RP_IDP_SIGN_KEY seems to work for authentication. -# Trying HS256 produces a "JWS token verification failed" error for some reason. +# Using RS256 + JWKS Endpoint seems to work with no value for OIDC_RP_IDP_SIGN_KEY +# seems to work for authentication. Trying HS256 produces a "JWS token verification failed" +# error for some reason. OIDC_RP_SIGN_ALGO = "RS256" OIDC_STORE_ACCESS_TOKEN = True OIDC_STORE_ID_TOKEN = True -# Security/access control connector. -# Currently one of 'ispyb' or 'ssh_ispyb'. -SECURITY_CONNECTOR = os.environ.get('SECURITY_CONNECTOR', 'ispyb').lower() -# Number of minutes to cache security information for a user. -# Set to '0' to disable caching. -SECURITY_CONNECTOR_CACHE_MINUTES = int( - os.environ.get('SECURITY_CONNECTOR_CACHE_MINUTES', '2') -) - # SessionRefresh configuration. # There's only one item - the token expiry period, with a default of 15 minutes. # The default is 15 minutes if you don't set this value. TOKEN_EXPIRY_MINUTES = os.environ.get("OIDC_RENEW_ID_TOKEN_EXPIRY_MINUTES", "15") OIDC_RENEW_ID_TOKEN_EXPIRY_SECONDS = int(TOKEN_EXPIRY_MINUTES) * 60 -# Keycloak mozilla_django_oidc - Settings - End - -# The deployment mode. -# Controls the behaviour of the application (it's strictness to errors etc). -# Typically one of "DEVELOPMENT" or "PRODUCTION". -# see api.utils for the 'deployment_mode_is_production()' function. -DEPLOYMENT_MODE = os.environ.get("DEPLOYMENT_MODE", "production").upper() - -# Authentication check when uploading files. -# This can be switched off to simplify development testing if required. -# It's asserted as True for 'production' mode. -AUTHENTICATE_UPLOAD = True -if os.environ.get("AUTHENTICATE_UPLOAD") == 'False': - assert DEPLOYMENT_MODE != "PRODUCTION" - AUTHENTICATE_UPLOAD = False - -ROOT_URLCONF = "fragalysis.urls" - -STATIC_ROOT = os.path.join(PROJECT_ROOT, "static") - -TEMPLATES = [ - { - "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [], - "APP_DIRS": True, - "OPTIONS": { - "context_processors": [ - "django.template.context_processors.debug", - "django.template.context_processors.request", - "django.contrib.auth.context_processors.auth", - "django.contrib.messages.context_processors.messages", - ] - }, - } -] WSGI_APPLICATION = "fragalysis.wsgi.application" -# Database -# https://docs.djangoproject.com/en/1.11/ref/settings/#databases - -CHEMCENTRAL_DB_NAME = os.environ.get("CHEMCENT_DB_NAME", "UNKOWN") - -DATABASE_ROUTERS = ['xchem_db.routers.AuthRouter'] +DATABASE_ROUTERS = ["xchem_db.routers.AuthRouter"] DATABASES = { "default": { @@ -294,9 +340,9 @@ } } -if os.environ.get("BUILD_XCDB") == 'yes': +if os.environ.get("BUILD_XCDB") == "yes": DATABASES["xchem_db"] = { - "ENGINE": 'django.db.backends.postgresql', + "ENGINE": "django.db.backends.postgresql", "NAME": os.environ.get("XCHEM_NAME", ""), "USER": os.environ.get("XCHEM_USER", ""), "PASSWORD": os.environ.get("XCHEM_PASSWORD", ""), @@ -304,7 +350,8 @@ "PORT": os.environ.get("XCHEM_PORT", ""), } -if CHEMCENTRAL_DB_NAME != "UNKOWN": +CHEMCENTRAL_DB_NAME = os.environ.get("CHEMCENT_DB_NAME", "UNKNOWN") +if CHEMCENTRAL_DB_NAME != "UNKNOWN": DATABASES["chemcentral"] = { "ENGINE": "django.db.backends.postgresql", "NAME": CHEMCENTRAL_DB_NAME, @@ -314,40 +361,14 @@ "PORT": 5432, } -# Password validation -# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" - }, - {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, - {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, - {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, -] - -# Internationalization -# https://docs.djangoproject.com/en/1.11/topics/i18n/ - -LANGUAGE_CODE = "en-us" - -TIME_ZONE = "UTC" - USE_I18N = True - USE_L10N = True - USE_TZ = True # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.11/howto/static-files/ - STATIC_URL = "/static/" MEDIA_ROOT = "/code/media/" MEDIA_URL = "/media/" -# Swagger loging / logout -LOGIN_URL = "/accounts/login/" -LOGOUT_URL = "/accounts/logout/" WEBPACK_LOADER = { "DEFAULT": { @@ -361,69 +382,13 @@ GRAPH_MODELS = {"all_applications": True, "group_models": True} # email settings for upload key stuff -EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' -EMAIL_HOST_USER = os.environ.get("EMAIL_USER") -# If there is an email user is defined then check the rest of the configuration is present. -# The defaults are set for the current (gamil) production configuration. -if EMAIL_HOST_USER: - EMAIL_HOST = os.environ.get('EMAIL_HOST', 'smtp.gmail.com') - EMAIL_USE_TLS = os.environ.get('EMAIL_USE_TLS', True) - EMAIL_PORT = os.environ.get('EMAIL_PORT', 587) +EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" +if EMAIL_HOST_USER := os.environ.get("EMAIL_USER"): + EMAIL_HOST = os.environ.get("EMAIL_HOST", "smtp.gmail.com") + EMAIL_USE_TLS = os.environ.get("EMAIL_USE_TLS", True) + EMAIL_PORT = os.environ.get("EMAIL_PORT", 587) EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_PASSWORD") - -# DOCS_ROOT = "/code/docs/_build/html " - -# Discourse settings for API calls to Discourse Platform -DISCOURSE_PARENT_CATEGORY = 'Fragalysis targets' -DISCOURSE_USER = 'fragalysis' -DISCOURSE_HOST = os.environ.get('DISCOURSE_HOST') -# Note that this can be obtained from discourse for the desired environment. -DISCOURSE_API_KEY = os.environ.get("DISCOURSE_API_KEY") - -# This suffix can be set to that the different development environments posting to the same Discourse -# server can "automatically" generate different category/post titles - hopefully reducing confusion. -# It will be appended at category or post-title, e.g. "Mpro-duncan", "Mpro-staging" etc. -# Note that it is for dev systems. It is not required on production because production will have a -# dedicated Discourse server. -DISCOURSE_DEV_POST_SUFFIX = os.environ.get("DISCOURSE_DEV_POST_SUFFIX", '') - -# An optional URL that identifies the URL to a prior stack. -# If set, it's typically something like "https://fragalysis.diamond.ac.uk". -# It can be blank, indicating there is no legacy service. -LEGACY_URL = os.environ.get("LEGACY_URL", "") - -SQUONK2_MEDIA_DIRECTORY = "fragalysis-files" -SQUONK2_INSTANCE_API = "data-manager-ui/results/instance/" - -# The Target Access String (TAS) Python regular expression. -# The Project title (the TAS) must match this expression to be valid. -# See api/utils.py validate_tas() for the current implementation. -# To simplify error messages when the match fails you can also -# add an error message. -TAS_REGEX = os.environ.get("TAS_REGEX", r"^(lb\d{5})(-(\d+)){0,1}$") -TAS_REGEX_ERROR_MSG = os.environ.get( - "TAS_REGEX_ERROR_MSG", - "Must begin 'lb' followed by 5 digits, optionally followed by a hyphen and a number.", -) -# Are any public target access strings defined? -# If so they'll be in the PUBLIC_TAS variable as a comma separated list. -PUBLIC_TAS = os.environ.get("PUBLIC_TAS", "") -PUBLIC_TAS_LIST = PUBLIC_TAS.split(",") if PUBLIC_TAS else [] - -COMPUTED_SET_MEDIA_DIRECTORY = "computed_set_data" -TARGET_LOADER_MEDIA_DIRECTORY = "target_loader_data" - -# A list of identifiers of messages generated by the system check framework -# that we wish to permanently acknowledge and ignore. -# Silenced checks will not be output to the console. -# -# fields.W342 Is issued for the xchem-db package. -# The hint is "ForeignKey(unique=True) is usually better served by a OneToOneField." -SILENCED_SYSTEM_CHECKS = [ - "fields.W342", -] - # Configure django logging. # We provide a standard formatter that emits a timestamp, the module issuing the log # and the level name, a little like this... @@ -433,36 +398,34 @@ # We provide a console and rotating file handler # (50Mi of logging in 10 files of 5M each), # with the rotating file handler typically used for everything. -DISABLE_LOGGING_FRAMEWORK = ( - True - if os.environ.get("DISABLE_LOGGING_FRAMEWORK", "no").lower() in ["yes"] - else False -) +DISABLE_LOGGING_FRAMEWORK = os.environ.get( + "DISABLE_LOGGING_FRAMEWORK", "no" +).lower() in ["yes"] LOGGING_FRAMEWORK_ROOT_LEVEL = os.environ.get("LOGGING_FRAMEWORK_ROOT_LEVEL", "DEBUG") if not DISABLE_LOGGING_FRAMEWORK: LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'simple': { - 'format': '%(asctime)s %(name)s.%(funcName)s():%(lineno)s %(levelname)s # %(message)s', - 'datefmt': '%Y-%m-%dT%H:%M:%S%z', + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "simple": { + "format": "%(asctime)s %(name)s.%(funcName)s():%(lineno)s %(levelname)s # %(message)s", + "datefmt": "%Y-%m-%dT%H:%M:%S%z", } }, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'stream': sys.stdout, - 'formatter': 'simple', + "handlers": { + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "stream": sys.stdout, + "formatter": "simple", }, - 'rotating': { - 'level': 'DEBUG', - 'class': 'logging.handlers.RotatingFileHandler', - 'maxBytes': 5_000_000, - 'backupCount': 10, - 'filename': os.path.join(BASE_DIR, 'logs/backend.log'), - 'formatter': 'simple', + "rotating": { + "level": "DEBUG", + "class": "logging.handlers.RotatingFileHandler", + "maxBytes": 5_000_000, + "backupCount": 10, + "filename": os.path.join(BASE_DIR, "logs/backend.log"), + "formatter": "simple", }, }, 'loggers': { @@ -474,8 +437,173 @@ 'urllib3': {'level': 'WARNING'}, 'paramiko': {'level': 'WARNING'}, }, - 'root': { - 'level': LOGGING_FRAMEWORK_ROOT_LEVEL, - 'handlers': ['console', 'rotating'], + "root": { + "level": LOGGING_FRAMEWORK_ROOT_LEVEL, + "handlers": ["console", "rotating"], }, } + +# -------------------------------------------------------------------------------------- +# FRAGALYSIS SETTINGS +# -------------------------------------------------------------------------------------- +# With comprehensive comments where necessary to explain the setting's values. + +# The deployment mode. +# Controls the behaviour of the application (it's strictness to errors etc). +# Typically one of "DEVELOPMENT" or "PRODUCTION". +# see api.utils for the 'deployment_mode_is_production()' function. +DEPLOYMENT_MODE: str = os.environ.get("DEPLOYMENT_MODE", "production").upper() + +# Authentication check when uploading files. +# This can be switched off to simplify development testing if required. +# It's asserted as True for 'production' mode. +AUTHENTICATE_UPLOAD: bool = True +if os.environ.get("AUTHENTICATE_UPLOAD") == "False": + assert DEPLOYMENT_MODE != "PRODUCTION" + AUTHENTICATE_UPLOAD = False + +COMPUTED_SET_MEDIA_DIRECTORY: str = "computed_set_data" + +# Discourse settings for API calls to Discourse Platform +DISCOURSE_PARENT_CATEGORY: str = "Fragalysis targets" +DISCOURSE_USER: str = "fragalysis" +DISCOURSE_HOST: str = os.environ.get("DISCOURSE_HOST", "") +# Note that this can be obtained from discourse for the desired environment. +DISCOURSE_API_KEY: str = os.environ.get("DISCOURSE_API_KEY", "") +# This suffix can be set to that the different development environments posting +# to the same Discourse server can "automatically" generate different category/post +# titles - hopefully reducing confusion. It will be appended at category or post-title, +# e.g. "Mpro-duncan", "Mpro-staging" etc. Note that it is for dev systems. +# It is not required on production because production will have a +# dedicated Discourse server. +DISCOURSE_DEV_POST_SUFFIX: str = os.environ.get("DISCOURSE_DEV_POST_SUFFIX", "") + +# Some Squonk2 developer/debug variables. +# Unused in production. +DUMMY_TARGET_TITLE: str = os.environ.get("DUMMY_TARGET_TITLE", "") +DUMMY_USER: str = os.environ.get("DUMMY_USER", "") +DUMMY_TAS: str = os.environ.get("DUMMY_TAS", "") + +# Do we enable the collection and presentation +# of the availability of underlying services? +# A colon (:) separated list of services to enable. +# See "viewer/services.py" for the full list of supported services. +ENABLE_SERVICE_STATUS: str = os.environ.get("ENABLE_SERVICE_STATUS", "") + +# What infection have been set? +# "Infections" are built-in faults that can be induced by providing their names. +# Typically these are "hard to reproduce" errors that are useful for testing. +# The names are provided in a comma-separated list in this variable. +# The full set of supported names can be used can be found in "api/infections.py" +INFECTIONS: str = os.environ.get("INFECTIONS", "").lower() + +# The ISpyB database settings. +# Can be used in conjunction with SSH settings (later in this file) +ISPYB_USER: str = os.environ.get("ISPYB_USER", "") +ISPYB_PASSWORD: str = os.environ.get("ISPYB_PASSWORD", "") +ISPYB_HOST: str = os.environ.get("ISPYB_HOST", "") +ISPYB_PORT: str = os.environ.get("ISPYB_PORT", "") + +# An optional URL that identifies the URL to a prior stack. +# If set, it's typically something like "https://fragalysis.diamond.ac.uk". +# It can be blank, indicating there is no legacy service. +LEGACY_URL: str = os.environ.get("LEGACY_URL", "") + +NEOMODEL_NEO4J_BOLT_URL: str = os.environ.get( + "NEO4J_BOLT_URL", "bolt://neo4j:test@neo4j:7687" +) + +# The graph (neo4j) database settings. +# The query provides the graph endpoint, typically a service in a kubernetes namespace +# like 'graph.graph-a.svc' and the 'auth' provides the graph username and password. +NEO4J_QUERY: str = os.environ.get("NEO4J_QUERY", "neo4j") +NEO4J_AUTH: str = os.environ.get("NEO4J_AUTH", "neo4j/neo4j") + +# These flags are used in the upload_tset form as follows. +# Proposal Supported | Proposal Required | Proposal / View fields +# Y | Y | Shown / Required +# Y | N | Shown / Optional +# N | N | Not Shown +PROPOSAL_SUPPORTED: bool = True +PROPOSAL_REQUIRED: bool = True + +# Are any public target access strings defined? +# If so they'll be in the PUBLIC_TAS variable as a comma separated list. +PUBLIC_TAS: str = os.environ.get("PUBLIC_TAS", "") +PUBLIC_TAS_LIST: List[str] = PUBLIC_TAS.split(",") if PUBLIC_TAS else [] + +# Security/access control connector. +# Currently one of 'ispyb' or 'ssh_ispyb'. +SECURITY_CONNECTOR: str = os.environ.get("SECURITY_CONNECTOR", "ispyb").lower() +# Number of minutes to cache security information for a user. +# Set to '0' to disable caching. +SECURITY_CONNECTOR_CACHE_MINUTES: int = int( + os.environ.get("SECURITY_CONNECTOR_CACHE_MINUTES", "2") +) + +# An SSH host. +# Used in the security module in conjunction with ISPyB settings. +# The SSH_PRIVATE_KEY_FILENAME value will be used if there is no SSH_PASSWORD. +SSH_HOST: str = os.environ.get("SSH_HOST", "") +SSH_USER: str = os.environ.get("SSH_USER", "") +SSH_PASSWORD: str = os.environ.get("SSH_PASSWORD", "") +SSH_PRIVATE_KEY_FILENAME: str = os.environ.get("SSH_PRIVATE_KEY_FILENAME", "") + +# The maximum length of the 'slug' used for names this Fragalysis will create. +# +# Squonk2 variables are generally used by the 'squonk2_agent.py' module +# in the 'viewer' package. +SQUONK2_MAX_SLUG_LENGTH: int = 10 + +# Where the Squonk2 logic places its files in Job containers. +SQUONK2_MEDIA_DIRECTORY: str = "fragalysis-files" +# The Squonk2 DataManger UI endpoint to obtain Job Instance information. +SQUONK2_INSTANCE_API: str = "data-manager-ui/results/instance/" + +# The URL for the Squonk2 Account Server API. +SQUONK2_ASAPI_URL: str = os.environ.get("SQUONK2_ASAPI_URL", "") +# The URL for the Squonk2 Data Manaqer API. +SQUONK2_DMAPI_URL: str = os.environ.get("SQUONK2_DMAPI_URL", "") +# The URL for the Squonk2 User Interface. +SQUONK2_UI_URL: str = os.environ.get("SQUONK2_UI_URL", "") +# The pre-assigned Squonk2 Account Server Organisation for the stack. +# This is created by an administrator of the Squonk2 service. +SQUONK2_ORG_UUID: str = os.environ.get("SQUONK2_ORG_UUID", "") +# The Account Server Unit billing day 9for all products (projects) that are created. +# It's a day of the month (1..27). +SQUONK2_UNIT_BILLING_DAY: str = os.environ.get("SQUONK2_UNIT_BILLING_DAY", "") +# The Squonk2 Account Server product "flavour" created for Jobs (products/projects). +# It's usually one of "GOLD", "SILVER" or "BRONZE". +SQUONK2_PRODUCT_FLAVOUR: str = os.environ.get("SQUONK2_PRODUCT_FLAVOUR", "") +# A short slug used when creating Squonk2 objects for this stack. +# This must be unique across all stacks that share the same Squonk2 service. +SQUONK2_SLUG: str = os.environ.get("SQUONK2_SLUG", "")[:SQUONK2_MAX_SLUG_LENGTH] +# The pre-assigned Squonk2 Account Server Organisation owner and password. +# This account is used to create Squonk2 objects for the stack. +SQUONK2_ORG_OWNER: str = os.environ.get("SQUONK2_ORG_OWNER", "") +SQUONK2_ORG_OWNER_PASSWORD: str = os.environ.get("SQUONK2_ORG_OWNER_PASSWORD", "") +# Do we verify Squonk2 SSL certificates ("yes" or "no"). +SQUONK2_VERIFY_CERTIFICATES: str = os.environ.get("SQUONK2_VERIFY_CERTIFICATES", "") + +TARGET_LOADER_MEDIA_DIRECTORY: str = "target_loader_data" + +# The Target Access String (TAS) Python regular expression. +# The Project title (the TAS) must match this expression to be valid. +# See api/utils.py validate_tas() for the current implementation. +# To simplify error messages when the match fails you can also +# add an error message. +TAS_REGEX: str = os.environ.get("TAS_REGEX", r"^(lb\d{5})(-(\d+)){0,1}$") +TAS_REGEX_ERROR_MSG: str = os.environ.get( + "TAS_REGEX_ERROR_MSG", + "Must begin 'lb' followed by 5 digits, optionally followed by a hyphen and a number.", +) + +# Version variables. +# These are set by the Dockerfile in the fragalysis-stack repository +# and controlled by the CI process, i.e. they're not normally set by a a user. +BE_NAMESPACE: str = os.environ.get("BE_NAMESPACE", "undefined") +BE_IMAGE_TAG: str = os.environ.get("BE_IMAGE_TAG", "undefined") +FE_NAMESPACE: str = os.environ.get("FE_NAMESPACE", "undefined") +FE_IMAGE_TAG: str = os.environ.get("FE_IMAGE_TAG", "undefined") +STACK_NAMESPACE: str = os.environ.get("STACK_NAMESPACE", "undefined") +STACK_VERSION: str = os.environ.get("STACK_VERSION", "undefined") diff --git a/fragalysis/views.py b/fragalysis/views.py index c68a4d4e..7b14e912 100644 --- a/fragalysis/views.py +++ b/fragalysis/views.py @@ -1,6 +1,4 @@ # Classes/Methods to override default OIDC Views (Keycloak authentication) -import os - from django.conf import settings from django.http import JsonResponse from mozilla_django_oidc.views import OIDCLogoutView @@ -34,41 +32,12 @@ def version(request): # Unused args del request - undefined_value = "undefined" - # b/e, f/e and stack origin comes form container environment variables. - # - # We also need to deal with empty or unset strings - # so the get() default does not help - be_namespace = os.environ.get('BE_NAMESPACE') - if not be_namespace: - be_namespace = undefined_value - - be_image_tag = os.environ.get('BE_IMAGE_TAG') - if not be_image_tag: - be_image_tag = undefined_value - - fe_namespace = os.environ.get('FE_NAMESPACE') - if not fe_namespace: - fe_namespace = undefined_value - - fe_branch = os.environ.get('FE_BRANCH') - if not fe_branch: - fe_branch = undefined_value - - stack_namespace = os.environ.get('STACK_NAMESPACE') - if not stack_namespace: - stack_namespace = undefined_value - - stack_version = os.environ.get('STACK_VERSION') - if not stack_version: - stack_version = undefined_value - version_response = { - 'version': { - 'backend': f'{be_namespace}:{be_image_tag}', - 'frontend': f'{fe_namespace}:{fe_branch}', - 'stack': f'{stack_namespace}:{stack_version}', + "version": { + "backend": f"{settings.BE_NAMESPACE}:{settings.BE_IMAGE_TAG}", + "frontend": f"{settings.FE_NAMESPACE}:{settings.FE_IMAGE_TAG}", + "stack": f"{settings.STACK_NAMESPACE}:{settings.STACK_VERSION}", } } return JsonResponse(version_response) diff --git a/network/views.py b/network/views.py index 8d790247..0be98a9d 100644 --- a/network/views.py +++ b/network/views.py @@ -1,5 +1,4 @@ -import os - +from django.conf import settings from django.http import HttpResponse from frag.network.decorate import get_add_del_link from frag.network.query import get_full_graph @@ -8,13 +7,9 @@ def full_graph(request): - """ - Get the full graph for a molecule from an input smiles - :param request: - :return: - """ - graph_choice = os.environ.get("NEO4J_QUERY", "neo4j") - graph_auth = os.environ.get("NEO4J_AUTH", "neo4j/neo4j") + """Get the full graph for a molecule from an input smiles""" + graph_choice = settings.NEO4J_QUERY + graph_auth = settings.NEO4J_AUTH if "graph_choice" in request.GET: graph_choice = request.GET["graph_choice"] if "smiles" in request.GET: diff --git a/poetry.lock b/poetry.lock index 64ceed5f..0fcf5b82 100644 --- a/poetry.lock +++ b/poetry.lock @@ -539,43 +539,43 @@ jinja2 = "*" [[package]] name = "cryptography" -version = "42.0.0" +version = "42.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:c640b0ef54138fde761ec99a6c7dc4ce05e80420262c20fa239e694ca371d434"}, - {file = "cryptography-42.0.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:678cfa0d1e72ef41d48993a7be75a76b0725d29b820ff3cfd606a5b2b33fda01"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:146e971e92a6dd042214b537a726c9750496128453146ab0ee8971a0299dc9bd"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87086eae86a700307b544625e3ba11cc600c3c0ef8ab97b0fda0705d6db3d4e3"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0a68bfcf57a6887818307600c3c0ebc3f62fbb6ccad2240aa21887cda1f8df1b"}, - {file = "cryptography-42.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5a217bca51f3b91971400890905a9323ad805838ca3fa1e202a01844f485ee87"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ca20550bb590db16223eb9ccc5852335b48b8f597e2f6f0878bbfd9e7314eb17"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:33588310b5c886dfb87dba5f013b8d27df7ffd31dc753775342a1e5ab139e59d"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9515ea7f596c8092fdc9902627e51b23a75daa2c7815ed5aa8cf4f07469212ec"}, - {file = "cryptography-42.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:35cf6ed4c38f054478a9df14f03c1169bb14bd98f0b1705751079b25e1cb58bc"}, - {file = "cryptography-42.0.0-cp37-abi3-win32.whl", hash = "sha256:8814722cffcfd1fbd91edd9f3451b88a8f26a5fd41b28c1c9193949d1c689dc4"}, - {file = "cryptography-42.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:a2a8d873667e4fd2f34aedab02ba500b824692c6542e017075a2efc38f60a4c0"}, - {file = "cryptography-42.0.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:8fedec73d590fd30c4e3f0d0f4bc961aeca8390c72f3eaa1a0874d180e868ddf"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be41b0c7366e5549265adf2145135dca107718fa44b6e418dc7499cfff6b4689"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ca482ea80626048975360c8e62be3ceb0f11803180b73163acd24bf014133a0"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c58115384bdcfe9c7f644c72f10f6f42bed7cf59f7b52fe1bf7ae0a622b3a139"}, - {file = "cryptography-42.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:56ce0c106d5c3fec1038c3cca3d55ac320a5be1b44bf15116732d0bc716979a2"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:324721d93b998cb7367f1e6897370644751e5580ff9b370c0a50dc60a2003513"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:d97aae66b7de41cdf5b12087b5509e4e9805ed6f562406dfcf60e8481a9a28f8"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:85f759ed59ffd1d0baad296e72780aa62ff8a71f94dc1ab340386a1207d0ea81"}, - {file = "cryptography-42.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:206aaf42e031b93f86ad60f9f5d9da1b09164f25488238ac1dc488334eb5e221"}, - {file = "cryptography-42.0.0-cp39-abi3-win32.whl", hash = "sha256:74f18a4c8ca04134d2052a140322002fef535c99cdbc2a6afc18a8024d5c9d5b"}, - {file = "cryptography-42.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:14e4b909373bc5bf1095311fa0f7fcabf2d1a160ca13f1e9e467be1ac4cbdf94"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3005166a39b70c8b94455fdbe78d87a444da31ff70de3331cdec2c568cf25b7e"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:be14b31eb3a293fc6e6aa2807c8a3224c71426f7c4e3639ccf1a2f3ffd6df8c3"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:bd7cf7a8d9f34cc67220f1195884151426ce616fdc8285df9054bfa10135925f"}, - {file = "cryptography-42.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c310767268d88803b653fffe6d6f2f17bb9d49ffceb8d70aed50ad45ea49ab08"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bdce70e562c69bb089523e75ef1d9625b7417c6297a76ac27b1b8b1eb51b7d0f"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e9326ca78111e4c645f7e49cbce4ed2f3f85e17b61a563328c85a5208cf34440"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:69fd009a325cad6fbfd5b04c711a4da563c6c4854fc4c9544bff3088387c77c0"}, - {file = "cryptography-42.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:988b738f56c665366b1e4bfd9045c3efae89ee366ca3839cd5af53eaa1401bce"}, - {file = "cryptography-42.0.0.tar.gz", hash = "sha256:6cf9b76d6e93c62114bd19485e5cb003115c134cf9ce91f8ac924c44f8c8c3f4"}, + {file = "cryptography-42.0.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be"}, + {file = "cryptography-42.0.2-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529"}, + {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9"}, + {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2"}, + {file = "cryptography-42.0.2-cp37-abi3-win32.whl", hash = "sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee"}, + {file = "cryptography-42.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee"}, + {file = "cryptography-42.0.2-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90"}, + {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea"}, + {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33"}, + {file = "cryptography-42.0.2-cp39-abi3-win32.whl", hash = "sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635"}, + {file = "cryptography-42.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2"}, + {file = "cryptography-42.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a"}, + {file = "cryptography-42.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65"}, + {file = "cryptography-42.0.2.tar.gz", hash = "sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888"}, ] [package.dependencies] diff --git a/viewer/cset_upload.py b/viewer/cset_upload.py index f1e84b6f..bb4e704f 100644 --- a/viewer/cset_upload.py +++ b/viewer/cset_upload.py @@ -198,9 +198,12 @@ def get_site_observation( zfile_hashvals=zfile_hashvals, ) else: - name = f'{compound_set.target.title}-{pdb_fn}' + name = pdb_fn try: - site_obvs = SiteObservation.objects.get(code__contains=name) + site_obvs = SiteObservation.objects.get( + code__contains=name, + experiment__experiment_upload__target__title=target, + ) except SiteObservation.DoesNotExist: # Initial SiteObservation lookup failed. logger.warning( @@ -210,7 +213,10 @@ def get_site_observation( ) # Try alternatives. # If all else fails then the site_obvs will be 'None' - qs = SiteObservation.objects.filter(code__contains=name) + qs = SiteObservation.objects.filter( + code__contains=name, + experiment__experiment_upload__target__title=target, + ) if qs.exists(): logger.info( 'Found SiteObservation containing name=%s qs=%s', @@ -219,7 +225,10 @@ def get_site_observation( ) else: alt_name = name.split(':')[0].split('_')[0] - qs = SiteObservation.objects.filter(code__contains=alt_name) + qs = SiteObservation.objects.filter( + code__contains=alt_name, + experiment__experiment_upload__target__title=target, + ) if qs.exists(): logger.info( 'Found SiteObservation containing alternative name=%s qs=%s', @@ -328,15 +337,13 @@ def set_mol( # try exact match first try: site_obvs = SiteObservation.objects.get( - code__contains=str(compound_set.target.title + '-' + i), + code=str(i), experiment__experiment_upload__target_id=compound_set.target, ) ref = site_obvs except SiteObservation.DoesNotExist: qs = SiteObservation.objects.filter( - code__contains=str( - compound_set.target.title + '-' + i.split(':')[0].split('_')[0] - ), + code=str(i.split(':')[0].split('_')[0]), experiment__experiment_upload__target_id=compound_set.target, ) if not qs.exists(): @@ -356,7 +363,7 @@ def set_mol( # Try to get the LHS SiteObservation, # This will be used to set the ComputedMolecule.site_observation_code. # This may fail. - lhs_property = 'lhs_pdb' + lhs_property = 'ref_pdb' lhs_so = self.get_site_observation( lhs_property, mol, @@ -503,7 +510,7 @@ def set_descriptions( computed_set.save() description_dict = description_mol.GetPropsAsDict() - for key in list(description_dict.keys()): + for key in description_dict.keys(): if key in descriptions_needed and key not in [ 'ref_mols', 'ref_pdb', diff --git a/viewer/download_structures.py b/viewer/download_structures.py index 65f94efc..59e7ceb9 100644 --- a/viewer/download_structures.py +++ b/viewer/download_structures.py @@ -11,6 +11,7 @@ import shutil import uuid import zipfile +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from io import BytesIO from pathlib import Path @@ -49,6 +50,13 @@ 'readme': (''), } + +@dataclass(frozen=True) +class ArchiveFile: + path: str + archive_path: str + + # Dictionary containing all references needed to create the zip file # NB you may need to add a version number to this at some point... zip_template = { @@ -216,7 +224,7 @@ def _read_and_patch_molecule_name(path, molecule_name=None): return content -def _add_file_to_zip_aligned(ziparchive, code, filepath): +def _add_file_to_zip_aligned(ziparchive, code, archive_file): """Add the requested file to the zip archive. If the file is an SDF or MOL we insert the name of the molecule @@ -230,39 +238,32 @@ def _add_file_to_zip_aligned(ziparchive, code, filepath): Returns: [boolean]: [True of record added to archive] """ - logger.debug('+_add_file_to_zip_aligned: %s, %s', code, filepath) - if not filepath: + logger.debug('+_add_file_to_zip_aligned: %s, %s', code, archive_file) + if not archive_file: # Odd - assume success logger.error('No filepath value') return True - # Incoming filepath can be both str and FieldFile - try: - filepath = filepath.path - except AttributeError: - filepath = str(Path(settings.MEDIA_ROOT).joinpath(filepath)) - - # strip off the leading parts of path - archive_path = str(Path(*Path(filepath).parts[7:])) + filepath = str(Path(settings.MEDIA_ROOT).joinpath(archive_file.path)) if Path(filepath).is_file(): if _is_mol_or_sdf(filepath): # It's a MOL or SD file. # Read and (potentially) adjust the file # and add to the archive as a string. content = _read_and_patch_molecule_name(filepath, molecule_name=code) - ziparchive.writestr(archive_path, content) + ziparchive.writestr(archive_file.archive_path, content) else: # Copy the file without modification - ziparchive.write(filepath, archive_path) + ziparchive.write(filepath, archive_file.archive_path) return True else: logger.warning('filepath "%s" is not a file', filepath) - _add_empty_file(ziparchive, archive_path) + _add_empty_file(ziparchive, archive_file.archive_path) return False -def _add_file_to_sdf(combined_sdf_file, filepath): +def _add_file_to_sdf(combined_sdf_file, archive_file): """Append the requested sdf file to the single sdf file provided. Args: @@ -274,19 +275,19 @@ def _add_file_to_sdf(combined_sdf_file, filepath): """ media_root = settings.MEDIA_ROOT - if not filepath: + if not archive_file.path: # Odd - assume success logger.error('No filepath value') return True - fullpath = os.path.join(media_root, filepath) + fullpath = os.path.join(media_root, archive_file.path) if os.path.isfile(fullpath): with open(combined_sdf_file, 'a', encoding='utf-8') as f_out: patched_sdf_content = _read_and_patch_molecule_name(fullpath) f_out.write(patched_sdf_content) return True else: - logger.warning('filepath "%s" is not a file', filepath) + logger.warning('filepath "%s" is not a file', archive_file.path) return False @@ -301,11 +302,8 @@ def _protein_files_zip(zip_contents, ziparchive, error_file): continue for prot, prot_file in files.items(): - # if it's a list of files (map_info) instead of single file - if not isinstance(prot_file, list): - prot_file = [prot_file] for f in prot_file: - if not _add_file_to_zip_aligned(ziparchive, prot.split(":")[0], f): + if not _add_file_to_zip_aligned(ziparchive, prot, f): error_file.write(f'{param},{prot},{f}\n') prot_errors += 1 @@ -333,14 +331,14 @@ def _molecule_files_zip(zip_contents, ziparchive, combined_sdf_file, error_file) ] is True and not _add_file_to_zip_aligned( ziparchive, prot.split(":")[0], file ): - error_file.write(f'sdf_info,{prot},{file}\n') + error_file.write(f'sdf_info,{prot},{file.path}\n') mol_errors += 1 # Append sdf file on the Molecule record to the combined_sdf_file. if zip_contents['molecules'][ 'single_sdf_file' ] is True and not _add_file_to_sdf(combined_sdf_file, file): - error_file.write(f'single_sdf_file,{prot},{file}\n') + error_file.write(f'single_sdf_file,{prot},{file.path}\n') mol_errors += 1 return mol_errors @@ -448,6 +446,46 @@ def _extra_files_zip(ziparchive, target): logger.info('Processed %s extra files', num_processed) +def _yaml_files_zip(ziparchive, target): + """Add all yaml files (except transforms) from upload to ziparchive""" + + for experiment_upload in target.experimentupload_set.order_by('commit_datetime'): + yaml_paths = ( + Path(settings.MEDIA_ROOT) + .joinpath(settings.TARGET_LOADER_MEDIA_DIRECTORY) + .joinpath(experiment_upload.task_id) + ) + + transforms = [ + Path(f.name).name + for f in ( + experiment_upload.neighbourhood_transforms, + experiment_upload.neighbourhood_transforms, + experiment_upload.neighbourhood_transforms, + ) + ] + # taking the latest upload for now + # add unpacked zip directory + yaml_paths = [d for d in list(yaml_paths.glob("*")) if d.is_dir()][0] + + # add upload_[d] dir + yaml_paths = next(yaml_paths.glob("upload_*")) + + archive_path = Path('yaml_files').joinpath(yaml_paths.parts[-1]) + + yaml_files = [ + f + for f in list(yaml_paths.glob("*.yaml")) + if f.is_file() and f.name not in transforms + ] + + logger.info('Processing yaml files (%s)...', yaml_files) + + for file in yaml_files: + logger.info('Adding yaml file "%s"...', file) + ziparchive.write(file, str(Path(archive_path).joinpath(file.name))) + + def _document_file_zip(ziparchive, download_path, original_search, host): """Create the document file This consists of a template plus an added contents description. @@ -583,6 +621,8 @@ def _create_structures_zip(target, zip_contents, file_url, original_search, host _extra_files_zip(ziparchive, target) + _yaml_files_zip(ziparchive, target) + _document_file_zip(ziparchive, download_path, original_search, host) error_file.close() @@ -625,21 +665,79 @@ def _create_structures_dict(target, site_obvs, protein_params, other_params): for so in site_obvs: for param in protein_params: if protein_params[param] is True: - try: - # getting the param from experiment. more data are - # coming from there, that's why this is in try - # block + if param in ['pdb_info', 'mtz_info', 'cif_info', 'map_info']: + # experiment object model_attr = getattr(so.experiment, param) - # getattr retrieves FieldFile object, hence the .name - if isinstance(model_attr, list): - # except map_files, this returns a list of files - zip_contents['proteins'][param][so.code] = model_attr + logger.debug( + 'Adding param to zip: %s, value: %s', param, model_attr + ) + if param != 'map_info': + # treat all params as list + model_attr = ( + [model_attr.name] + # None - some weird glitch in storing the values + if model_attr and not str(model_attr).find('None') > -1 + else [param] + ) + + afile = [] + for f in model_attr: + # here the model_attr is already stringified + if model_attr and model_attr != 'None': + archive_path = str( + Path('crystallographic_files') + .joinpath(so.code) + .joinpath( + Path(f) + .parts[-1] + .replace(so.experiment.code, so.code) + ) + ) + else: + archive_path = param + afile.append(ArchiveFile(path=f, archive_path=archive_path)) + + elif param in [ + 'bound_file', + 'apo_solv_file', + 'apo_desolv_file', + 'apo_file', + 'sigmaa_file', + 'event_file', + 'artefacts_file', + 'pdb_header_file', + 'diff_file', + ]: + # siteobservation object + + model_attr = getattr(so, param) + logger.debug( + 'Adding param to zip: %s, value: %s', param, model_attr + ) + if model_attr and model_attr != 'None': + archive_path = str( + Path('aligned_files') + .joinpath(so.code) + .joinpath( + Path(model_attr.name) + .parts[-1] + .replace(so.longcode, so.code) + ) + ) else: - zip_contents['proteins'][param][so.code] = model_attr.name + archive_path = param + + afile = [ + ArchiveFile( + path=model_attr.name, + archive_path=archive_path, + ) + ] + else: + logger.warning('Unexpected param: %s', param) + continue - except AttributeError: - # on the off chance that the data are in site_observation model - zip_contents['proteins'][param][so.code] = getattr(so, param).name + zip_contents['proteins'][param][so.code] = afile if other_params['single_sdf_file'] is True: zip_contents['molecules']['single_sdf_file'] = True @@ -666,7 +764,14 @@ def _create_structures_dict(target, site_obvs, protein_params, other_params): if rel_sd_file: logger.debug('rel_sd_file=%s code=%s', rel_sd_file, so.code) - zip_contents['molecules']['sdf_files'].update({rel_sd_file: so.code}) + zip_contents['molecules']['sdf_files'].update( + { + ArchiveFile( + path=rel_sd_file, + archive_path=rel_sd_file, + ): so.code + } + ) num_molecules_collected += 1 # Report (in the log) anomalies diff --git a/viewer/managers.py b/viewer/managers.py index 836ff422..7a1a4826 100644 --- a/viewer/managers.py +++ b/viewer/managers.py @@ -17,6 +17,7 @@ def filter_qs(self): ).annotate( target=F("experiment__experiment_upload__target"), compound_code=F("cmpd__compound_code"), + prefix_tooltip=F("experiment__prefix_tooltip"), ) return qs diff --git a/viewer/migrations/0043_experiment_prefix_tooltip.py b/viewer/migrations/0043_experiment_prefix_tooltip.py new file mode 100644 index 00000000..93477ed4 --- /dev/null +++ b/viewer/migrations/0043_experiment_prefix_tooltip.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.23 on 2024-02-13 15:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('viewer', '0042_alter_xtalformsite_xtalform_site_num'), + ] + + operations = [ + migrations.AddField( + model_name='experiment', + name='prefix_tooltip', + field=models.TextField(null=True), + ), + ] diff --git a/viewer/models.py b/viewer/models.py index e3839475..c2b8af72 100644 --- a/viewer/models.py +++ b/viewer/models.py @@ -196,6 +196,7 @@ class Experiment(models.Model): map_info = ArrayField(models.FileField(max_length=255), null=True) type = models.PositiveSmallIntegerField(null=True) pdb_sha256 = models.TextField(null=True) + prefix_tooltip = models.TextField(null=True) compounds = models.ManyToManyField( "Compound", through="ExperimentCompound", diff --git a/viewer/sdf_check.py b/viewer/sdf_check.py index 949f1001..411128e4 100755 --- a/viewer/sdf_check.py +++ b/viewer/sdf_check.py @@ -89,10 +89,12 @@ def check_refmol(mol, validate_dict, target=None): for ref_mol in ref_mols: ref_strip = ref_mol.strip() - query_string = f'{target}-' + ref_strip.split(':')[0].split('_')[0] - query = SiteObservation.objects.filter(code__contains=query_string) + query = SiteObservation.objects.filter( + code=ref_strip, + experiment__experiment_upload__target__title=target, + ) if len(query) == 0: - msg = f"No SiteObservation code contains '{query_string}'" + msg = f"No SiteObservation code contains '{ref_strip}'" validate_dict = add_warning( molecule_name=mol.GetProp('_Name'), field='ref_mol', diff --git a/viewer/serializers.py b/viewer/serializers.py index c969d3da..8694419f 100644 --- a/viewer/serializers.py +++ b/viewer/serializers.py @@ -465,8 +465,8 @@ class Meta: class GraphSerializer(serializers.ModelSerializer): graph = serializers.SerializerMethodField() - graph_choice = os.environ.get("NEO4J_QUERY", "neo4j") - graph_auth = os.environ.get("NEO4J_AUTH", "neo4j/neo4j") + graph_choice = settings.NEO4J_QUERY + graph_auth = settings.NEO4J_AUTH def get_graph(self, obj): return get_full_graph( @@ -951,6 +951,7 @@ class Meta: class SiteObservationReadSerializer(serializers.ModelSerializer): compound_code = serializers.StringRelatedField() + prefix_tooltip = serializers.StringRelatedField() class Meta: model = models.SiteObservation diff --git a/viewer/services.py b/viewer/services.py index a203bfc3..77417143 100644 --- a/viewer/services.py +++ b/viewer/services.py @@ -6,6 +6,7 @@ from enum import Enum import requests +from django.conf import settings from frag.utils.network_utils import get_driver from pydiscourse import DiscourseClient @@ -18,8 +19,8 @@ # Default timeout for any request calls REQUEST_TIMEOUT_S = 5 -_NEO4J_LOCATION: str = os.environ.get("NEO4J_QUERY", "neo4j") -_NEO4J_AUTH: str = os.environ.get("NEO4J_AUTH", "neo4j/neo4j") +_NEO4J_LOCATION: str = settings.NEO4J_QUERY +_NEO4J_AUTH: str = settings.NEO4J_AUTH class State(str, Enum): diff --git a/viewer/squonk2_agent.py b/viewer/squonk2_agent.py index 3d4f7936..aadab6c0 100644 --- a/viewer/squonk2_agent.py +++ b/viewer/squonk2_agent.py @@ -12,6 +12,7 @@ from urllib.parse import ParseResult, urlparse import requests +from django.conf import settings from requests import Response from squonk2.as_api import AsApi, AsApiRv from squonk2.auth import Auth @@ -58,9 +59,7 @@ # How long are Squonk2 'names'? _SQ2_MAX_NAME_LENGTH: int = 80 -# A slug used for names this Fragalysis will create -# and a prefix string. So Squonk2 objects will be called "Fragalysis {slug}" -_MAX_SLUG_LENGTH: int = 10 +# An object prefix string. So Squonk2 objects will be called "Fragalysis {slug}" _SQ2_NAME_PREFIX: str = "Fragalysis" # Built-in @@ -94,46 +93,24 @@ def __init__(self): # "Fragalysis {SLUG} ", this leaves (80-22) 58 characters for the # use with the target-access-string and session project strings # to form Squonk2 Unit and Project names. - self.__CFG_SQUONK2_ASAPI_URL: Optional[str] = os.environ.get( - 'SQUONK2_ASAPI_URL' - ) - self.__CFG_SQUONK2_DMAPI_URL: Optional[str] = os.environ.get( - 'SQUONK2_DMAPI_URL' - ) - self.__CFG_SQUONK2_UI_URL: Optional[str] = os.environ.get('SQUONK2_UI_URL') - self.__CFG_SQUONK2_ORG_UUID: Optional[str] = os.environ.get('SQUONK2_ORG_UUID') - self.__CFG_SQUONK2_UNIT_BILLING_DAY: Optional[str] = os.environ.get( - 'SQUONK2_UNIT_BILLING_DAY' - ) - self.__CFG_SQUONK2_PRODUCT_FLAVOUR: Optional[str] = os.environ.get( - 'SQUONK2_PRODUCT_FLAVOUR' - ) - self.__CFG_SQUONK2_SLUG: Optional[str] = os.environ.get('SQUONK2_SLUG', '')[ - :_MAX_SLUG_LENGTH - ] - self.__CFG_SQUONK2_ORG_OWNER: Optional[str] = os.environ.get( - 'SQUONK2_ORG_OWNER' - ) - self.__CFG_SQUONK2_ORG_OWNER_PASSWORD: Optional[str] = os.environ.get( - 'SQUONK2_ORG_OWNER_PASSWORD' - ) - self.__CFG_OIDC_AS_CLIENT_ID: Optional[str] = os.environ.get( - 'OIDC_AS_CLIENT_ID' - ) - self.__CFG_OIDC_DM_CLIENT_ID: Optional[str] = os.environ.get( - 'OIDC_DM_CLIENT_ID' - ) - self.__CFG_OIDC_KEYCLOAK_REALM: Optional[str] = os.environ.get( - 'OIDC_KEYCLOAK_REALM' - ) + self.__CFG_SQUONK2_ASAPI_URL: str = settings.SQUONK2_ASAPI_URL + self.__CFG_SQUONK2_DMAPI_URL: str = settings.SQUONK2_DMAPI_URL + self.__CFG_SQUONK2_UI_URL: str = settings.SQUONK2_UI_URL + self.__CFG_SQUONK2_ORG_UUID: str = settings.SQUONK2_ORG_UUID + self.__CFG_SQUONK2_UNIT_BILLING_DAY: str = settings.SQUONK2_UNIT_BILLING_DAY + self.__CFG_SQUONK2_PRODUCT_FLAVOUR: str = settings.SQUONK2_PRODUCT_FLAVOUR + self.__CFG_SQUONK2_SLUG: str = settings.SQUONK2_SLUG + self.__CFG_SQUONK2_ORG_OWNER: str = settings.SQUONK2_ORG_OWNER + self.__CFG_SQUONK2_ORG_OWNER_PASSWORD: str = settings.SQUONK2_ORG_OWNER_PASSWORD + self.__CFG_OIDC_AS_CLIENT_ID: str = settings.OIDC_AS_CLIENT_ID + self.__CFG_OIDC_DM_CLIENT_ID: str = settings.OIDC_DM_CLIENT_ID + self.__CFG_OIDC_KEYCLOAK_REALM: str = settings.OIDC_KEYCLOAK_REALM # Optional config (no '__CFG_' prefix) - self.__DUMMY_TARGET_TITLE: Optional[str] = os.environ.get('DUMMY_TARGET_TITLE') - self.__DUMMY_USER: Optional[str] = os.environ.get('DUMMY_USER') - self.__DUMMY_TAS: Optional[str] = os.environ.get('DUMMY_TAS') - self.__SQUONK2_VERIFY_CERTIFICATES: Optional[str] = os.environ.get( - 'SQUONK2_VERIFY_CERTIFICATES' - ) + self.__DUMMY_TARGET_TITLE: str = settings.DUMMY_TARGET_TITLE + self.__DUMMY_USER: str = settings.DUMMY_USER + self.__DUMMY_TAS: str = settings.DUMMY_TAS + self.__SQUONK2_VERIFY_CERTIFICATES: str = settings.SQUONK2_VERIFY_CERTIFICATES # The integer billing day, valid if greater than zero self.__unit_billing_day: int = 0 @@ -799,9 +776,9 @@ def configured(self) -> Squonk2AgentRv: # Is the slug too long? # Limited to 10 characters assert self.__CFG_SQUONK2_SLUG - if len(self.__CFG_SQUONK2_SLUG) > _MAX_SLUG_LENGTH: + if len(self.__CFG_SQUONK2_SLUG) > settings.SQUONK2_MAX_SLUG_LENGTH: msg = ( - f'Slug is longer than {_MAX_SLUG_LENGTH} characters' + f'Slug is longer than {settings.SQUONK2_MAX_SLUG_LENGTH} characters' f' ({self.__CFG_SQUONK2_SLUG})' ) _LOGGER.error(msg) diff --git a/viewer/target_loader.py b/viewer/target_loader.py index b9665633..edad072e 100644 --- a/viewer/target_loader.py +++ b/viewer/target_loader.py @@ -700,6 +700,7 @@ def _enumerate_objects(self, objects: dict, attr: str) -> None: def process_experiment( self, item_data: tuple[str, dict] | None = None, + prefix_tooltips: dict[str, str] | None = None, validate_files: bool = True, **kwargs, ) -> ProcessedObject | None: @@ -734,6 +735,7 @@ def process_experiment( """ del kwargs assert item_data + assert prefix_tooltips logger.debug("incoming data: %s", item_data) experiment_name, data = item_data @@ -813,6 +815,9 @@ def process_experiment( # version int old versions are kept target loader version = 1 + code_prefix = extract(key="code_prefix") + prefix_tooltip = prefix_tooltips.get(code_prefix, "") + fields = { "code": experiment_name, } @@ -830,6 +835,7 @@ def process_experiment( "mtz_info": str(self._get_final_path(mtz_info)), "cif_info": str(self._get_final_path(cif_info)), "map_info": map_info_paths, + "prefix_tooltip": prefix_tooltip, # this doesn't seem to be present # pdb_sha256: } @@ -839,6 +845,7 @@ def process_experiment( index_fields = { "xtalform": assigned_xtalform, "smiles": smiles, + "code_prefix": code_prefix, } return ProcessedObject( @@ -1437,7 +1444,6 @@ def process_bundle(self): self.report.log(logging.ERROR, msg) raise KeyError(msg) from exc - # moved this bit from init self.target, target_created = Target.objects.get_or_create( title=self.target_name, display_name=self.target_name, @@ -1475,6 +1481,7 @@ def process_bundle(self): self.version_number = meta["version_number"] self.version_dir = meta["version_dir"] self.previous_version_dirs = meta["previous_version_dirs"] + prefix_tooltips = meta["code_prefix_tooltips"] # check transformation matrix files ( # pylint: disable=unbalanced-tuple-unpacking @@ -1533,7 +1540,9 @@ def process_bundle(self): ), ) - experiment_objects = self.process_experiment(yaml_data=crystals) + experiment_objects = self.process_experiment( + yaml_data=crystals, prefix_tooltips=prefix_tooltips + ) compound_objects = self.process_compound( yaml_data=crystals, experiments=experiment_objects ) @@ -1643,16 +1652,14 @@ def process_bundle(self): canon_site_confs=canon_site_conf_objects, ) - values = ["xtalform_site__xtalform", "canon_site_conf__canon_site", "cmpd"] + # values = ["canon_site_conf__canon_site", "cmpd"] + values = ["experiment"] qs = ( SiteObservation.objects.values(*values) .order_by(*values) .annotate(obvs=ArrayAgg("id")) .values_list("obvs", flat=True) ) - current_list = SiteObservation.objects.filter( - experiment__experiment_upload__target=self.target - ).values_list('code', flat=True) for elem in qs: # objects in this group should be named with same scheme so_group = SiteObservation.objects.filter(pk__in=elem) @@ -1681,20 +1688,27 @@ def process_bundle(self): # technically it should be validated in previous try-catch block logger.error("Non-standard SiteObservation code 2: %s", last) - logger.debug("iter_pos: %s", iter_pos) - # ... and create new one starting from next item suffix = alphanumerator(start_from=iter_pos) for so in so_group.filter(code__isnull=True): - code = f"{so.experiment.code.split('-')[1]}{next(suffix)}" + code_prefix = experiment_objects[so.experiment.code].index_data[ + "code_prefix" + ] + code = f"{code_prefix}{so.experiment.code.split('-')[1]}{next(suffix)}" # test uniqueness for target # TODO: this should ideally be solved by db engine, before # rushing to write the trigger, have think about the # loader concurrency situations - prefix = alphanumerator() - while code in current_list: - code = f"{next(prefix)}{code}" + if SiteObservation.objects.filter( + experiment__experiment_upload__target=self.target, + code=code, + ).exists(): + msg = ( + f"short code {code} already exists for this target; " + + "specify a code_prefix to resolve this conflict" + ) + self.report.log(logging.ERROR, msg) so.code = code so.save() @@ -1710,7 +1724,7 @@ def process_bundle(self): # tag site observations for val in canon_site_objects.values(): # pylint: disable=no-member - tag = f"{val.instance.canon_site_num} - {val.instance.name}" + tag = f"{val.instance.canon_site_num} - {''.join(val.instance.name.split('+')[1:-1])}" so_list = SiteObservation.objects.filter( canon_site_conf__canon_site=val.instance ) @@ -1725,7 +1739,7 @@ def process_bundle(self): tag = ( f"{val.instance.canon_site.canon_site_num}" + f"{next(numerators[val.instance.canon_site.canon_site_num])}" - + f" - {val.instance.name}" + + f" - {val.instance.name.split('+')[0]}" ) so_list = [ site_observation_objects[strip_version(k)].instance diff --git a/viewer/views.py b/viewer/views.py index 6112f7ba..b3e3562a 100644 --- a/viewer/views.py +++ b/viewer/views.py @@ -1492,7 +1492,7 @@ def create(self, request): # prot = models.Protein.objects.filter(code__contains=code_first_part).values() # I don't see why I need to drop out of django objects here prot = models.SiteObservation.objects.filter( - code__contains=code_first_part + experiment__experiment_upload__target=target, code=code_first_part ) if prot.exists(): # even more than just django object, I need an @@ -2487,7 +2487,7 @@ def get(self, *args, **kwargs): del args, kwargs logger.debug("+ ServiceServiceState.State.get called") - service_string = os.environ.get("ENABLE_SERVICE_STATUS", "") + service_string = settings.ENABLE_SERVICE_STATUS logger.debug("Service string: %s", service_string) services = [k for k in service_string.split(":") if k != ""]