chore: use the common network to access update redis & mongodb

.env

@@ -1,10 +1,10 @@
 COMPOSE_PATH_SEPARATOR=;
-COMPOSE_FILE=docker-compose.yml;docker/dev.yml;docker/mongodb.yml;docker/ml.yml
+COMPOSE_FILE=docker-compose.yml;docker/dev.yml;docker/ml.yml
 
 # Docker
 RESTART_POLICY=no
 # for dev only: connection to local product opener network
-PO_LOCAL_NET=po_default
+COMMON_NET_NAME=po_default
 
 # User uid - in dev align this with your own user uid / gid
 # see https://gist.github.com/alexgarel/6e6158ee869d6db2192e0441fd58576e
@@ -39,7 +39,6 @@ ELASTIC_USER=elastic
 ELASTIC_PASSWORD=elastic
 
 # Postgres
-# use network qualified name for dev for we have two networks
 POSTGRES_HOST=postgres
 POSTGRES_DB=postgres
 POSTGRES_USER=postgres
@@ -61,13 +60,16 @@ INFLUXDB_AUTH_TOKEN=
 # on linux, this will work if you have an influxdb listening on 0.0.0.0
 # INFLUXDB_HOST=host.docker.internal
 
-# MongoDB (dev settings, using robotoff-specific MongoDB instance by default)
+# MongoDB (dev settings, using shared MongoDB instance by default)
 # To use Product Opener instance, use following commented line
 # MONGO_URI=mongodb://mongodb.po_default:27017
 MONGO_URI=mongodb://mongodb:27017
 
-# Redis
+# Local Redis, used to manage rq jobs
 REDIS_HOST=redis
+# Redis where Product Opener publishes updates in the Redis Stream
+# We uses here the same Redis instance as the one used for rq jobs
+# for simplicity, but in production it uses a different one.
 REDIS_UPDATE_HOST=redis
 
 # OpenFoodFacts API

.github/workflows/container-deploy.yml

@@ -31,9 +31,6 @@ jobs:
         echo "MONGO_URI=mongodb://10.1.0.200:27017" >> $GITHUB_ENV
         # In staging, we use the Redis of shared services to get Product Update
         echo "REDIS_UPDATE_HOST=redis.po_webnet" >> $GITHUB_ENV
-        echo "REDIS_UPDATE_PORT=6379" >> $GITHUB_ENV
-        # And the local redis for the rest
-        echo "REDIS_HOST=redis.robotoff_default" >> $GITHUB_ENV
         echo "INFLUXDB_HOST=10.1.0.200" >> $GITHUB_ENV
         echo "IMAGE_MODERATION_SERVICE_URL=https://nutripatrol.openfoodfacts.net/api/v1/flags" >> $GITHUB_ENV
     - name: Set various variable for production deployment
@@ -46,7 +43,6 @@ jobs:
         echo "MONGO_URI=mongodb://10.1.0.113:27017" >> $GITHUB_ENV
         # In prod, we use the Redis running in VM
         echo "REDIS_UPDATE_HOST=10.1.0.113" >> $GITHUB_ENV
-        echo "REDIS_UPDATE_PORT=6379" >> $GITHUB_ENV
         echo "INFLUXDB_HOST=10.1.0.201" >> $GITHUB_ENV
         echo "IMAGE_MODERATION_SERVICE_URL=https://nutripatrol.openfoodfacts.org/api/v1/flags" >> $GITHUB_ENV
     - name: Wait for container build workflow
@@ -123,7 +119,15 @@ jobs:
           # Set app variables
           echo "ROBOTOFF_INSTANCE=${{ env.ROBOTOFF_INSTANCE }}" >> .env
           echo "ROBOTOFF_TLD=${{ env.ROBOTOFF_TLD }}" >> .env
+          # This is the network shared with Product Opener.
+          # In staging, Product Opener is deployed on the same VM.
+          # In production Product Opener is deployed on a separate server and is not dockerized.
+          echo "COMMON_NET_NAME=po_webnet" >> .env
           echo "REDIS_HOST=redis.robotoff_default" >> .env
+          echo "REDIS_UPDATE_HOST=${{ env.REDIS_UPDATE_HOST }}" >> .env
+          echo "REDIS_UPDATE_PORT=6379" >> .env
+          # We use local PostgreSQL instance with `robotoff_default` network,
+          # so that there is no possibility of conflict with PO PostgreSQL instance.
           echo "POSTGRES_HOST=postgres.robotoff_default" >> .env
           echo "POSTGRES_DB=postgres" >> .env
           echo "POSTGRES_USER=postgres" >> .env

Makefile

@@ -15,7 +15,7 @@ ENV_FILE ?= .env
 MOUNT_POINT ?= /mnt
 HOSTS=127.0.0.1 robotoff.openfoodfacts.localhost
 DOCKER_COMPOSE=docker compose --env-file=${ENV_FILE}
-DOCKER_COMPOSE_TEST=COMPOSE_PROJECT_NAME=robotoff_test PO_LOCAL_NET=po_test docker compose --env-file=${ENV_FILE}
+DOCKER_COMPOSE_TEST=COMPOSE_PROJECT_NAME=robotoff_test COMMON_NET_NAME=po_test docker compose --env-file=${ENV_FILE}
 ML_OBJECT_DETECTION_MODELS := tf-universal-logo-detector tf-nutrition-table tf-nutriscore
 
 # mount information for robotoff backup ZFS dataset
@@ -194,7 +194,7 @@ docs:
 	@echo "🥫 Generationg doc…"
 	${DOCKER_COMPOSE} run --rm --no-deps api ./build_mkdocs.sh
 
-checks: toml-check flake8 black-check mypy isort-check docs
+checks: create_external_networks toml-check flake8 black-check mypy isort-check docs
 
 lint: toml-lint isort black
 
@@ -250,7 +250,7 @@ create_external_volumes:
 
 create_external_networks:
 	@echo "🥫 Creating external networks if needed … (dev only)"
-	( docker network create ${PO_LOCAL_NET} || true )
+	( docker network create ${COMMON_NET_NAME} || true )
 # for tests
 	( docker network create po_test || true )
 

doc/how-to-guides/deployment/dev-install.md

@@ -11,8 +11,6 @@ You should, consider those changes:
 
   You should also run `make dl-models` to download all machine learning models locally.
 
-- if you don't work on Product Opener, you should launch MongoDB by adding `docker/mongodb.yml` to `COMPOSE_FILE`
-
 - change `OFF_UID` and `OFF_GID` to match your own user UID/GID (optional, only if you experience some file permission issue, see [Getting developper uid for docker](https://gist.github.com/alexgarel/6e6158ee869d6db2192e0441fd58576e))
 
 Note: **beware** not to commit your local changes to `.env` file!

docker-compose.yml

@@ -12,6 +12,9 @@ x-robotoff-base:
   # this image is built with target=runtime, so it won't have the dev dependencies
   image: ghcr.io/openfoodfacts/robotoff:${TAG}
   volumes: *robotoff-base-volumes
+  networks:
+    - default
+    - common_net
 
 x-robotoff-base-env:
   &robotoff-base-env
@@ -61,6 +64,10 @@ x-robotoff-worker-base:
     - postgres
     - redis
   mem_limit: 8g
+  networks:
+    - default
+    - common_net
+
 services:
   api:
     <<: *robotoff-base
@@ -168,3 +175,8 @@ volumes:
 
 networks:
   default:
+
+  # this is the network shared with product opener
+  common_net:
+    name: ${COMMON_NET_NAME}
+    external: true

docker/prod.yml

@@ -8,3 +8,9 @@ volumes:
   es-data:
     external: true
     name: ${COMPOSE_PROJECT_NAME:-robotoff}_es-data
+
+
+networks:
+  common_net:
+    name: ${COMMON_NET_NAME}
+    external: true