global: python3.10/11, opensearch, invenio upgrades, react-formule

.github/workflows/harbour-build-server.yml

@@ -2,7 +2,7 @@ name: Build and export CAP server image
 
 on:
   push:
-    branches: ['master', 'dev', 'qa', 'test']
+    branches: ["master", "dev", "qa", "test", "next"]
     paths-ignore:
       - "ui/**"
 

.github/workflows/harbour-build-ui.yml

@@ -2,7 +2,7 @@ name: Build and export CAP UI image
 
 on:
   push:
-    branches: ["master", "dev", "qa", "test", "with-formule"]
+    branches: ["master", "dev", "qa", "test", "with-formule", "next"]
     # paths:
     #   - "ui/**"
     #   - ".github/workflows/ui-test.yml"

Dockerfile

@@ -30,6 +30,9 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
+RUN apt-get update -q \
+    && apt-get install -y kstart krb5-kdc krb5-admin-server krb5-multidev
+
 # Install Invenio
 ENV WORKING_DIR=/opt/cap
 ENV INVENIO_INSTANCE_PATH=${WORKING_DIR}/var/instance
@@ -59,6 +62,7 @@ RUN mkdir -p ${INVENIO_INSTANCE_PATH}
 RUN mkdir -p ${CAP_FILES_DIR}
 
 RUN pip install -e .
+RUN pip install gunicorn
 
 RUN cat ./docker/base/CERN_Root_Certification_Authority_2.pem >>  /usr/local/lib/python3.10/site-packages/certifi/cacert.pem
 
@@ -70,7 +74,6 @@ ENV APP_GITHUB_OAUTH_ACCESS_TOKEN=${APP_GITHUB_OAUTH_ACCESS_TOKEN}
 ARG APP_GITLAB_OAUTH_ACCESS_TOKEN
 ENV APP_GITLAB_OAUTH_ACCESS_TOKEN=${APP_GITLAB_OAUTH_ACCESS_TOKEN}
 
-RUN pip install gunicorn
 
 # Set folder permissions
 RUN chgrp -R 0 ${WORKING_DIR} ${CAP_FILES_DIR} && \

MANIFEST.in

@@ -50,6 +50,7 @@ include docker/harbour/ui/Dockerfile
 include docker/base/Dockerfile
 include docker/haproxy/Dockerfile
 include docker/nginx/Dockerfile
+include docker/nginxdev/Dockerfile
 include docker/postgres/Dockerfile
 include scripts/bootstrap
 include scripts/console
@@ -93,3 +94,9 @@ recursive-include ui *.yarnclean
 recursive-include ui *.yml
 recursive-include ui *.editorconfig
 prune docs/_build
+
+# added by check-manifest
+recursive-include docker *.js
+recursive-include ui *.jsx
+recursive-include ui *.json
+recursive-include ui *.js

cap/config.py

@@ -22,9 +22,10 @@
 from invenio_deposit.utils import check_oauth2_scope
 from invenio_oauthclient.contrib.cern import REMOTE_APP as CERN_REMOTE_APP
 from invenio_oauthclient.contrib.cern_openid import REMOTE_APP as CERN_OPENID_REMOTE_APP
-from invenio_records_rest.config import RECORDS_REST_ENDPOINTS
+from invenio_records_rest.config import RECORDS_REST_ENDPOINTS, RECORDS_REST_DEFAULT_SORT
 from invenio_records_rest.facets import range_filter, terms_filter
 from invenio_records_rest.utils import allow_all, deny_all
+from invenio_search.utils import prefix_index
 from jsonresolver import JSONResolver
 from jsonresolver.contrib.jsonref import json_loader_factory
 
@@ -310,6 +311,17 @@ def _(x):
 }
 RECORDS_REST_SORT_OPTIONS.update(DEPOSIT_REST_SORT_OPTIONS)
 
+records_sort_options_keys = list(RECORDS_REST_SORT_OPTIONS.keys())
+for records_sort_options_key in records_sort_options_keys:
+    RECORDS_REST_SORT_OPTIONS[prefix_index(records_sort_options_key, prefix=os.environ.get('SEARCH_INDEX_PREFIX', ''))] = \
+        RECORDS_REST_SORT_OPTIONS[records_sort_options_key]
+
+records_sort_keys = list(RECORDS_REST_DEFAULT_SORT.keys())
+for records_sort_key in records_sort_keys:
+    RECORDS_REST_DEFAULT_SORT[prefix_index(records_sort_key, prefix=os.environ.get('SEARCH_INDEX_PREFIX', ''))] = \
+        RECORDS_REST_DEFAULT_SORT[records_sort_key]
+
+
 #: Record search facets.
 # for aggregations, only ones starting with facet_ will be displayed on a page
 CAP_FACETS = {
@@ -488,7 +500,10 @@ def _(x):
     },
 }
 
-RECORDS_REST_FACETS = {'deposits': CAP_FACETS, 'records': CAP_FACETS}
+RECORDS_REST_FACETS = {
+    prefix_index('deposits', prefix=os.environ.get('SEARCH_INDEX_PREFIX', ''), app=1): CAP_FACETS,
+    prefix_index('records', prefix=os.environ.get('SEARCH_INDEX_PREFIX', ''), app=1): CAP_FACETS
+}
 
 #: Records REST API endpoints.
 RECORDS_REST_ENDPOINTS = copy.deepcopy(RECORDS_REST_ENDPOINTS)
@@ -835,6 +850,7 @@ def _(x):
 # =======
 #: Flag for not replacing refs when creating deposit
 INDEXER_REPLACE_REFS = False
+INDEXER_RECORD_TO_INDEX = "cap.modules.schemas.utils._record_to_index"
 
 # LHCB DB files location
 # ======================

cap/modules/experiments/search/cms_triggers.py

@@ -27,6 +27,7 @@
 
 from opensearch_dsl import Q, Search
 from invenio_search.proxies import current_search_client as es
+from invenio_search.utils import prefix_index
 
 CMS_TRIGGERS_ES_CONFIG = {
     'alias': 'cms-triggers',
@@ -69,12 +70,13 @@ class CMSTriggerSearch(Search):
     class Meta:
         """Meta class."""
 
-        index = CMS_TRIGGERS_ES_CONFIG['alias']
         fields = ('trigger', )
 
     def __init__(self, **kwargs):
         """Use Meta to set kwargs defaults."""
-        kwargs.setdefault('index', getattr(self.Meta, 'index', None))
+        prefixed_index = prefix_index(CMS_TRIGGERS_ES_CONFIG['alias'])
+
+        kwargs.setdefault('index', prefixed_index)
         kwargs.setdefault('using', es)
 
         super(CMSTriggerSearch, self).__init__(**kwargs)

cap/modules/experiments/search/das.py

@@ -25,6 +25,7 @@
 
 from opensearch_dsl import Search
 from invenio_search.proxies import current_search_client as es
+from invenio_search.utils import prefix_index
 
 DAS_DATASETS_ES_CONFIG = {
     'alias': 'das-datasets',
@@ -55,12 +56,13 @@ class DASSearch(Search):
     class Meta:
         """Meta class."""
 
-        index = DAS_DATASETS_ES_CONFIG['alias']
         fields = ('name', )
 
     def __init__(self, **kwargs):
         """Use Meta to set kwargs defaults."""
-        kwargs.setdefault('index', getattr(self.Meta, 'index', None))
+        prefixed_index = prefix_index(DAS_DATASETS_ES_CONFIG['alias'])
+
+        kwargs.setdefault('index', prefixed_index)
         kwargs.setdefault('using', es)
 
         super(DASSearch, self).__init__(**kwargs)

cap/modules/experiments/utils/cms.py

@@ -32,6 +32,7 @@
 
 from flask import current_app
 from invenio_db import db
+from invenio_search.utils import prefix_index
 
 from cap.modules.deposit.errors import DepositDoesNotExist,\
     DepositValidationError
@@ -60,7 +61,7 @@ def cache_cms_triggers_in_es_from_file(source):
 
     :param source: list of dict with dataset, year and trigger
     """
-    recreate_es_index_from_source(alias=CMS_TRIGGERS_ES_CONFIG['alias'],
+    recreate_es_index_from_source(alias=prefix_index(CMS_TRIGGERS_ES_CONFIG['alias']),
                                   mapping=CMS_TRIGGERS_ES_CONFIG['mappings'],
                                   settings=CMS_TRIGGERS_ES_CONFIG['settings'],
                                   source=source)

cap/modules/experiments/utils/common.py

@@ -34,6 +34,7 @@
 from opensearchpy import helpers
 from flask import current_app
 from invenio_search.proxies import current_search_client as es
+from invenio_search.utils import prefix_index
 
 
 def kinit(principal, keytab):
@@ -136,10 +137,16 @@ def _batches(iterable, chunk_size=10):
         for _, g in groupby(iterable, lambda _: next(c) // chunk_size):
             yield g
 
-    if es.indices.exists('{}-v1'.format(alias)):
-        old_index, new_index = ('{}-v1'.format(alias), '{}-v2'.format(alias))
+    if es.indices.exists(prefix_index('{}-v1'.format(alias))):
+        old_index, new_index = (
+            prefix_index('{}-v1'.format(alias)),
+            prefix_index('{}-v2'.format(alias))
+        )
     else:
-        old_index, new_index = ('{}-v2'.format(alias), '{}-v1'.format(alias))
+        old_index, new_index = (
+            prefix_index('{}-v2'.format(alias)),
+            prefix_index('{}-v1'.format(alias))
+        )
 
     # recreate new index
     if es.indices.exists(new_index):

cap/modules/experiments/utils/das.py

@@ -23,6 +23,8 @@
 # as an Intergovernmental Organization or submit itself to any jurisdiction.
 """Cern Analysis Preservation methods for DAS database connection."""
 
+from invenio_search.utils import prefix_index
+
 from ..search.das import DAS_DATASETS_ES_CONFIG
 from .common import recreate_es_index_from_source
 
@@ -42,7 +44,7 @@ def update_term_for_das_query(term, suffix=None):
 def cache_das_datasets_in_es_from_file(source):
     """Cache datasets names from DAS in ES."""
     recreate_es_index_from_source(source=source,
-                                  alias=DAS_DATASETS_ES_CONFIG['alias'],
+                                  alias=prefix_index(DAS_DATASETS_ES_CONFIG['alias']),
                                   mapping=DAS_DATASETS_ES_CONFIG['mappings'],
                                   settings=DAS_DATASETS_ES_CONFIG['settings'],
                                   id_getter=lambda obj: obj['name'])

cap/modules/schemas/cli.py

@@ -40,6 +40,7 @@
 from invenio_jsonschemas.errors import JSONSchemaNotFound
 from invenio_oauth2server.models import Token
 from invenio_search import current_search_client
+from invenio_search.utils import prefix_index
 from jsonschema import Draft7Validator, exceptions
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
@@ -377,7 +378,7 @@ def validate(
 
     # get all the records for this specific schema/type combination
     records = current_search_client.search(
-        index=f"{search_path}-{schema.name}-v{schema.version}",
+        index=prefix_index(f"{search_path}-{schema.name}-v{schema.version}"),
         body={
             "query": {
                 "bool": {

cap/modules/schemas/models.py

@@ -36,6 +36,7 @@
 from invenio_rest.errors import FieldError
 from invenio_search import current_search
 from invenio_search import current_search_client as es
+from invenio_search.utils import prefix_index
 from jsonschema import Draft7Validator
 from six.moves.urllib.parse import urljoin
 from sqlalchemy import UniqueConstraint, event
@@ -508,14 +509,15 @@ def name_to_es_name(name):
 
 def create_index(index_name, mapping_body, aliases):
     """Create index in opensearch, add under given aliases."""
-    if not es.indices.exists(index_name):
-        current_search.mappings[index_name] = {}  # invenio search needs it
+    prefixed_index = prefix_index(index_name)
+    if not es.indices.exists(prefixed_index):
+        current_search.mappings[prefixed_index] = {}  # invenio search needs it
 
-        es.indices.create(index=index_name, body=mapping_body, ignore=False)
+        es.indices.create(index=prefixed_index, body=mapping_body, ignore=False)
 
         for alias in aliases:
             es.indices.update_aliases(
-                {'actions': [{'add': {'index': index_name, 'alias': alias}}]}
+                {'actions': [{'add': {'index': prefixed_index, 'alias': prefix_index(alias)}}]}
             )
 
 
@@ -558,7 +560,10 @@ def after_insert_schema(target, value, schema):
 def before_delete_schema(mapper, connect, schema):
     """On schema delete, delete corresponding indexes and aliases in ES."""
     if schema.is_indexed:
-        for index in (schema.record_index, schema.deposit_index):
+        for index in (
+            prefix_index(schema.record_index), 
+            prefix_index(schema.deposit_index)
+        ):
             if es.indices.exists(index):
                 es.indices.delete(index)
 

cap/modules/schemas/utils.py

@@ -29,6 +29,8 @@
 from flask import abort, jsonify
 from invenio_jsonschemas.errors import JSONSchemaNotFound
 from jsonpatch import JsonPatchConflict
+from invenio_indexer.utils import default_record_to_index
+from invenio_search.utils import prefix_index
 
 from .models import Schema
 from .permissions import AdminSchemaPermission, ReadSchemaPermission
@@ -239,3 +241,14 @@ def wrapper(
         )
 
     return wrapper
+
+def _record_to_index(record):
+    """Get index given a record.
+
+    It tries to extract from `record['$schema']` the index.
+    If it fails, return the default value.
+    :param record: The record object.
+    :returns: The index.
+    """
+    index = default_record_to_index(record)
+    return prefix_index(index)

docker-compose.e2e-override-ui.yml

@@ -9,7 +9,7 @@ version: "2.3"
 services:
   nginx:
     restart: "always"
-    image: gitlab-registry.cern.ch/analysispreservation/nginx-ui:ant-e2e
+    image: gitlab-registry.cern.ch/analysispreservation/nginx-ui:next-e2e
     ports:
       - "80:80"
       - "443:443"
@@ -18,6 +18,7 @@ services:
     volumes:
       - ./docker/nginx/conf.d:/etc/nginx/conf.d
       - ./docker/nginx/ssl:/etc/ssl
+      - ./docker/nginx/config.js:/usr/share/nginx/html/config.js
   cypress:
     image: "cypress/included:12.10.0"
     # entrypoint: cypress run --headless --browser chrome
@@ -43,7 +44,7 @@ services:
     extends:
       file: docker-services.yml
       service: app
-    command: uwsgi /opt/cap/var/instance/uwsgi_rest.ini --py-autoreload=1 --http-timeout=3800 --socket-timeout=3800 --honour-stdin
+    command: gunicorn -b 0.0.0.0:5000 cap.wsgi:application
     ports:
       - "5000"
     volumes:
@@ -80,7 +81,7 @@ services:
     extends:
       file: docker-services.yml
       service: app
-    command: uwsgi /opt/cap/var/instance/uwsgi_rest.ini --py-autoreload=1 --http-timeout=3800 --socket-timeout=3800 --honour-stdin
+    command: gunicorn -b 0.0.0.0:5000 cap.wsgi:application
     ports:
       - "5000"
     environment:

docker-compose.e2e.yml

@@ -17,6 +17,10 @@ services:
     depends_on:
       - web-api
       - frontend
+    volumes:
+      - ./docker/nginx/conf.d:/etc/nginx/conf.d
+      - ./docker/nginx/ssl:/etc/ssl
+      - ./docker/nginx/config.js:/usr/share/nginx/html/config.js
   cypress:
     image: "cypress/included:12.10.0"
     # entrypoint: cypress run --headless --browser chrome

docker-services.yml

@@ -18,7 +18,7 @@ services:
   app-image:
     extends:
       service: app-base
-    image: gitlab-registry.cern.ch/analysispreservation/analysispreservation-openshift/app:master-e2e
+    image: gitlab-registry.cern.ch/analysispreservation/analysispreservation-openshift/app:next-e2e
   app-base:
     environment:
       - "INVENIO_ACCOUNTS_SESSION_REDIS_URL=redis://cache:6379/1"
@@ -40,6 +40,7 @@ services:
       - "CACHE_REDIS_HOST=cache"
       - "INVENIO_CACHE_REDIS_HOST=cache"
       - "INVENIO_ACCESS_SESSION_REDIS_HOST=cache"
+      - "INVENIO_SEARCH_INDEX_PREFIX=docker-"
   lb:
     build: ./docker/haproxy/
     image: cap-lb

docker/nginx/Dockerfile

@@ -12,7 +12,7 @@ RUN apt-get update
 RUN apt-get install -y curl bash
 
 
-RUN curl -sL https://deb.nodesource.com/setup_14.x  | bash -
+RUN curl -sL https://deb.nodesource.com/setup_18.x  | bash -
 RUN apt-get install -y nodejs
 
 RUN apt-get update && apt-get -y install  git python g++ make