Update to auto-deploy and allow jobs to run on catalog-beta

.github/workflows/ckan.yml

@@ -16,7 +16,7 @@ on:
         description: 'App to run on:'
         required: true
         type: string
-        default: 'catalog-admin'
+        default: 'catalog-next-admin'
       command:
         description: 'Command to run:'
         required: true

.github/workflows/ckan_auto.yml

@@ -34,7 +34,7 @@ jobs:
           #   - Create Informational issue: tracking-update-info.md
           #  schedule: sitemap update
           #   - Only run on [prod]
-          #   - Run on [catalog-gather] app
+          #   - Run on [catalog-admin] app
           #  schedule: db-solr-sync-next
           #   - Only run on [staging, prod]
           #   - Create Error issue: if runtime longer than 30 mins
@@ -44,7 +44,7 @@ jobs:
             "schedule": ["${{env.SCHEDULE_TRACKING}}", "${{env.SCHEDULE_SITEMAP}}",
                          "${{env.SCHEDULE_DBSOLR_SYNC}}"],
             "environ": ["development", "staging", "prod"],
-            "include": [ {"app": "catalog-admin"},
+            "include": [ {"app": "catalog-next-admin"},
                          {"error_seconds": 22000},
                          {"info_issue": false},
                          {"issue_template": ".github/automated_ckan_error.md"},
@@ -54,7 +54,7 @@ jobs:
                          {"schedule": "${{env.SCHEDULE_TRACKING}}", "info_issue": true},
                          {"schedule": "${{env.SCHEDULE_TRACKING}}", "issue_template": ".github/tracking-update-info.md"},
                          {"schedule": "${{env.SCHEDULE_SITEMAP}}", "command": "ckan geodatagov sitemap-to-s3"},
-                         {"schedule": "${{env.SCHEDULE_SITEMAP}}", "app": "catalog-gather"},
+                         {"schedule": "${{env.SCHEDULE_SITEMAP}}", "app": "catalog-next-admin"},
                          {"schedule": "${{env.SCHEDULE_DBSOLR_SYNC}}", "command": "ckan geodatagov db-solr-sync-next"},
                          {"schedule": "${{env.SCHEDULE_DBSOLR_SYNC}}", "error_seconds": 1800},
                          {"schedule": "${{env.SCHEDULE_DBSOLR_SYNC}}", "info_issue": true},

.github/workflows/commit.yml

@@ -58,18 +58,19 @@
       environ: development
     secrets: inherit
 
-  deploy-development:
-    if: github.ref == 'refs/heads/develop'
-    name: deploy (development)
-    needs:
-      - create-cloudgov-services-development
-    uses: gsa/data.gov/.github/workflows/deploy-template.yml@main
-    with:
-      environ: development
-      app_url: https://catalog-dev-datagov.app.cloud.gov
-      app_names: "{\"include\":[{\"app\":\"catalog-web\",\"smoketest\":true},{\"app\":\"catalog-admin\"},{\"app\":\"catalog-fetch\"},{\"app\":\"catalog-gather\"},{\"app\":\"catalog-proxy\",\"robots-dev\":true}]}"
-    secrets: inherit
+  # deploy-development:
+  #   if: github.ref == 'refs/heads/develop'
+  #   name: deploy (development)
+  #   needs:
+  #     - create-cloudgov-services-development
+  #   uses: gsa/data.gov/.github/workflows/deploy-template.yml@main
+  #   with:
+  #     environ: development
+  #     app_url: https://catalog-dev-datagov.app.cloud.gov
+  #     app_names: "{\"include\":[{\"app\":\"catalog-web\",\"smoketest\":true},{\"app\":\"catalog-admin\"},{\"app\":\"catalog-fetch\"},{\"app\":\"catalog-gather\"},{\"app\":\"catalog-proxy\",\"robots-dev\":true}]}"
+  #   secrets: inherit
 
+# TODO: change this branch back to develop
   # catalog-next
   deploy-next-development:
     if: github.ref == 'refs/heads/catalog-next'

.github/workflows/egress_check.yml

@@ -20,10 +20,8 @@ jobs:
           - 'BadDomainTest || curl -I -L --http1.1 https://yahoo.com | grep \"HTTP/1.1 403 Forbidden\"'
         environ: ['development', 'staging', 'prod']
         app:
-          - 'catalog-admin'
-          - 'catalog-web'
-          - 'catalog-fetch'
-          - 'catalog-gather'
+          - 'catalog-next-admin'
+          - 'catalog-next-web'
 
     steps:
       - name: Store Instance Name

.github/workflows/publish.yml

@@ -4,7 +4,7 @@ name: 2 - Publish & Deploy
 on:
   push:
     branches:
-      - main
+      - catalog-next
 
 jobs:
   ghcr_publish:
@@ -17,13 +17,13 @@ jobs:
         include:
           - image: ckan
             name: catalog.data.gov
-            tag: ghcr.io/gsa/catalog.data.gov:latest
+            tag: ghcr.io/gsa/catalog-beta.data.gov:latest
           - image: solr
             name: catalog.data.gov.solr
-            tag: ghcr.io/gsa/catalog.data.gov.solr:latest
+            tag: ghcr.io/gsa/catalog-beta.data.gov.solr:latest
           - image: postgresql
             name: catalog.data.gov.db
-            tag: ghcr.io/gsa/catalog.data.gov.db:latest
+            tag: ghcr.io/gsa/catalog-beta.data.gov.db:latest
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -62,7 +62,7 @@ jobs:
     with:
       environ: staging
       app_url: https://catalog-stage-datagov.app.cloud.gov
-      app_names: "{\"include\":[{\"app\":\"catalog-web\",\"smoketest\":true},{\"app\":\"catalog-admin\"},{\"app\":\"catalog-fetch\"},{\"app\":\"catalog-gather\"},{\"app\":\"catalog-proxy\"}]}"
+      app_names: "{\"include\":[{\"app\":\"catalog-next-web\",\"smoketest\":true},{\"app\":\"catalog-next-admin\"},{\"app\":\"catalog-next-proxy\"}]}"
     secrets: inherit
 
   deploy-prod:
@@ -74,7 +74,7 @@ jobs:
     with:
       environ: prod
       app_url: https://catalog.data.gov
-      app_names: "{\"include\":[{\"app\":\"catalog-web\",\"smoketest\":true},{\"app\":\"catalog-admin\"},{\"app\":\"catalog-fetch\"},{\"app\":\"catalog-gather\"},{\"app\":\"catalog-proxy\"}]}"
+      app_names: "{\"include\":[{\"app\":\"catalog-next-web\",\"smoketest\":true},{\"app\":\"catalog-next-admin\"},{\"app\":\"catalog-next-proxy\"}]}"
     secrets: inherit
 
   network-policies:
@@ -88,8 +88,8 @@ jobs:
       fail-fast: false
       matrix:
         command: [
-          "cf add-network-policy catalog-proxy catalog-web --protocol tcp --port 61443",
-          "cf add-network-policy catalog-proxy catalog-admin --protocol tcp --port 61443"
+          "cf add-network-policy catalog-next-proxy catalog-next-web --protocol tcp --port 61443",
+          "cf add-network-policy catalog-next-proxy catalog-next-admin --protocol tcp --port 61443"
         ]
         environ: ["staging", "prod"]
     steps:

.github/workflows/restart.yml

@@ -17,13 +17,13 @@ jobs:
     uses: gsa/data.gov/.github/workflows/app-restart-template.yml@main
     with:
       environ: staging
-      app_names: "{\"include\":[{\"app\":\"catalog-proxy\"},{\"app\":\"catalog-gather\"},{\"app\":\"catalog-fetch\"},{\"app\":\"catalog-web\"},{\"app\":\"catalog-admin\"},]}"
+      app_names: "{\"include\":[{\"app\":\"catalog-next-proxy\"},{\"app\":\"catalog-next-web\"},{\"app\":\"catalog-next-admin\"},]}"
     secrets: inherit
 
   restart-prod:
     name: restart (prod)
     uses: gsa/data.gov/.github/workflows/app-restart-template.yml@main
     with:
       environ: prod
-      app_names: "{\"include\":[{\"app\":\"catalog-proxy\"},{\"app\":\"catalog-gather\"},{\"app\":\"catalog-fetch\"},{\"app\":\"catalog-web\"},{\"app\":\"catalog-admin\"},]}"
+      app_names: "{\"include\":[{\"app\":\"catalog-next-proxy\"},{\"app\":\"catalog-next-web\"},{\"app\":\"catalog-next-admin\"},]}"
     secrets: inherit

.github/workflows/scale_web.yml

@@ -12,6 +12,6 @@ jobs:
     uses: gsa/data.gov/.github/workflows/scale-web-template.yml@main
     with:
       environ: prod
-      app_url: https://catalog.data.gov
-      app_names: "{\"include\":[{\"app\":\"catalog-web\",\"smoketest\":true},]}"
+      app_url: https://catalog-beta.data.gov
+      app_names: "{\"include\":[{\"app\":\"catalog-next-web\",\"smoketest\":true},]}"
     secrets: inherit

.github/workflows/snyk.yml

@@ -4,7 +4,7 @@ name: Check for Snyk Vulnerabilities
 on:
   pull_request:
     branches:
-      - main
+      - catalog-next
   workflow_dispatch:
   schedule:
     - cron: '0 12 * * *'  # every day at 12pm UTC

ckan/requirements.in

@@ -1,7 +1,7 @@
 # CKAN requirements and extensions
 git+https://github.com/GSA/[email protected]#egg=ckan
-git+https://github.com/ckan/ckanext-dcat@v1.7.0#egg=ckanext-dcat
--e git+https://github.com/ckan/ckanext-spatial.git@v2.3.0#egg=ckanext-spatial
+git+https://github.com/ckan/ckanext-dcat@v2.4.0#egg=ckanext-dcat
+-e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext_spatial
 git+https://github.com/GSA/ckanext-saml2auth.git@ckan-2-11-datagov#egg=ckanext-saml2auth
 
 -e git+https://github.com/GSA/ckanext-datagovcatalog.git@harvest-next#egg=ckanext_datagovcatalog
@@ -21,10 +21,6 @@ pika>=1.1.0,<1.3.0
 # requests>=2.11.1 # included in ckan core
 # six>=1.12.0 # included in dcat
 
-# ckanext-archiver
-# https://github.com/ckan/ckanext-archiver/blob/master/requirements.txt
-progressbar2==3.53.3
-
 # ckanext-geodatagov
 # https://github.com/GSA/ckanext-geodatagov/blob/main/requirements.txt
 git+https://github.com/danizen/PyZ3950#egg=PyZ3950
@@ -46,10 +42,10 @@ xlrd==2.0.1
 messytables==0.15.2
 
 #ckanext-dcat
-rdflib==6.1.1
-# rdflib-jsonld==0.4.0 # ignoring as cannot build on cloud.gov
+rdflib>=6.1.1
+pyld>=2.0.4
 geomet>=0.2.0
-future>=0.18.3
+# ckantoolkit>=0.0.7 # duplicated in ckanext-harvest dependencies
 
 # ckan doesn't advertise its requirements correctly, so let's add them here.
 # https://github.com/ckan/ckan/blob/ckan-2.9.5/requirements.txt
@@ -60,7 +56,7 @@ blinker==1.8.2
 certifi>=2024.7.4
 click==8.1.7
 dominate==2.9.1
-feedgen==1.0.0
+git+https://github.com/GSA/python-feedgen.git@main
 Flask==3.0.3
 Flask-Babel==4.0.0
 Flask-Login==0.6.3
@@ -80,7 +76,7 @@ pysolr==3.9.0
 python-dateutil==2.9.0.post0
 pytz
 pyyaml==6.0.1
-requests==2.32.3
+requests==2.32.4
 rq==1.16.2
 simplejson==3.19.2
 SQLAlchemy[mypy]==1.4.52
@@ -93,11 +89,8 @@ zope.interface==6.4post2
 
 
 # we are running under gunicorn
-# Pinned greenlet version to match version delivered with OS,
-# preventing error with gevent dependency using newer version.
 gunicorn
 eventlet
-# greenlet==0.4.12
 
 # New Relic
 newrelic
@@ -116,7 +109,12 @@ lxml==5.1.0
 numpy==1.26.4
 
 # snyk finding
-setuptools~=71.0.3
+setuptools>=78.1.1
+
+# avoid conflic dependencies issue
+greenlet>=3.1.1
+pip>=25.0
+urllib3>=2.5.0
 
 # Pin MarkupSafe to avoid button issue for logged in user
 MarkupSafe==2.*

ckan/requirements.txt

@@ -6,6 +6,7 @@ blinker==1.8.2
 boto3==1.40.23
 botocore==1.40.23
 cachelib==0.13.0
+cachetools==6.2.0
 certifi==2025.8.3
 cffi==1.17.1
 chardet==5.2.0
@@ -14,12 +15,12 @@ ckan @ git+https://github.com/GSA/ckan.git@416b88e74d2bd643bcdd1da7a900e29c6d443
 -e git+https://github.com/GSA/ckanext-datagovcatalog.git@b60f69e4cb15d3cdce80698e5b4b2dc406a2ca05#egg=ckanext_datagovcatalog
 ckanext-datagovtheme==0.4.0
 ckanext-datajson==0.1.28
-ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@b8ebf24004cd3f3edb7f9d01c87c20259c102093
+ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@255fcf1352273f9be98e9f02a5198efc4790c94e
 ckanext-envvars==0.0.6
 ckanext-geodatagov==0.4.0
 ckanext-metrics-dashboard==0.1.7
 ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@99f35585c219a5cd39717b8c42cc54cdd959dfb4
--e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext_spatial
+-e git+https://github.com/ckan/ckanext-spatial.git@938308469892e4bcf7389cb4adee5ccdd5a0ccca#egg=ckanext_spatial
 ckantoolkit==0.0.7
 click==8.1.7
 cryptography==45.0.7
@@ -28,13 +29,13 @@ dnspython==2.7.0
 dominate==2.9.1
 elementpath==5.0.4
 eventlet==0.40.3
-feedgen==1.0.0
+feedgen @ git+https://github.com/GSA/python-feedgen.git@b1fe34a7e14ebdaf6f415cdee5855c3c77305f16
 Flask==3.0.3
 flask-babel==4.0.0
 Flask-Login==0.6.3
 Flask-Session==0.8.0
 Flask-WTF==1.2.1
-future==1.0.0
+frozendict==2.4.6
 GeoAlchemy2==0.5.0
 geojson==3.1.0
 geomet==1.1.0
@@ -68,29 +69,28 @@ pip==25.2
 ply==3.11
 polib==1.2.0
 progressbar==2.5
-progressbar2==3.53.3
 psycopg2==2.9.9
 pycparser==2.22
 PyJWT==2.8.0
+PyLD==2.0.4
 pyOpenSSL==25.1.0
 pyparsing==3.1.2
 pyproj==3.4.1
 pysaml2==7.0.1
 pysolr==3.9.0
 python-dateutil==2.9.0.post0
 python-magic==0.4.27
-python-utils==3.9.1
 pytz==2025.2
 PyYAML==6.0.1
 PyZ3950 @ git+https://github.com/danizen/PyZ3950@6d44a4ab85c8bda3a7542c2c9efdfad46c830219
-rdflib==6.1.1
+rdflib==7.1.4
 redis==6.4.0
-requests==2.32.3
+requests==2.32.4
 rfc3987==1.3.8
 rq==1.16.2
 s3transfer==0.13.1
 sansjson==0.3.0
-setuptools==71.0.4
+setuptools==80.9.0
 shapely==2.0.7
 simplejson==3.19.2
 six==1.17.0