omnibus: cleanups (#781)

* omnibus: cleanups * omnibus: update esg script * omnibus: further cleanups
finos · Nov 8, 2023 · b7fa89a · b7fa89a
1 parent 8d3b34b
commit b7fa89a
Show file tree

Hide file tree

Showing 21 changed files with 72 additions and 104 deletions.
diff --git a/installers/omnibus/.env b/installers/omnibus/.env
@@ -13,8 +13,10 @@ LEGEND_OMNIBUS_SDLC_VERSION=0.148.1
 LEGEND_OMNIBUS_PURE_IDE_VERSION=4.35.0
 LEGEND_OMNIBUS_STUDIO_VERSION=10.54.0
 
+# TODO: to be removed when we make SDLC backend configurable
+# See https://github.com/finos/legend-sdlc/issues/722
 LEGEND_OMNIBUS_SDLC_DEMO_IN_MEMORY_VERSION=latest
-LEGEND_OMNIBUS_SDLC_DEMO_FILE_SYSTEM_VERSION=0.141.0
+LEGEND_OMNIBUS_SDLC_DEMO_FILE_SYSTEM_VERSION=0.148.1
 
 ####################################################################################
 
@@ -35,6 +37,7 @@ LEGEND_OMNIBUS_GITLAB_ROOT_PASSWORD=@iamLegendary
 
 # Legend Engine
 LEGEND_OMNIBUS_ENGINE_PORT=6300
+LEGEND_OMNIBUS_ENGINE_ADMIN_PORT=6301
 LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT=6310
 
 # Legend SDLC

diff --git a/installers/omnibus/components/engine/config/config.yml b/installers/omnibus/components/engine/config/config.yml
@@ -1,13 +1,19 @@
 server:
-  type: simple
-  applicationContextPath: /
   adminContextPath: /admin
-  connector:
-    maxRequestHeaderSize: 32KiB
-    type: http
-    port: ${LEGEND_OMNIBUS_ENGINE_PORT}
+  applicationConnectors:
+    - type: http
+      port: ${LEGEND_OMNIBUS_ENGINE_PORT}
+      maxRequestHeaderSize: 128KiB
+  adminConnectors:
+    - type: http
+      port: ${LEGEND_OMNIBUS_ENGINE_ADMIN_PORT}
+  gzip:
+    includedMethods:
+      - GET
+      - POST
   requestLog:
     appenders: []
+  rootPath: /api
 sessionCookie: LEGEND_ENGINE_JSESSIONID
 logging:
   level: INFO
@@ -17,14 +23,10 @@ logging:
 swagger:
   title: Legend Engine
   resourcePackage: org.finos.legend
-  uriPrefix: /api
 
 pac4j:
-  callbackPrefix: /api/pac4j
   clients:
     - org.pac4j.core.client.direct.AnonymousClient: {}
-  bypassPaths:
-    - /api/server/v1/info
 
 deployment:
   mode: TEST_IGNORE_FUNCTION_MATCH
@@ -44,8 +46,9 @@ metadataserver:
     port: 1234 # unsupported flow
     prefix: /depot/api
   sdlc:
-    host: legend-sdlc
-    port: ${LEGEND_OMNIBUS_SDLC_PORT}
+    host: localhost
+    port: ${LEGEND_OMNIBUS_NGINX_PORT}
+    prefix: /sdlc
 temporarytestdb:
   port: ${LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT}
 relationalexecution:

diff --git a/installers/omnibus/components/sdlc/config/config.file-system.yml b/installers/omnibus/components/sdlc/config/config.file-system.yml
@@ -32,12 +32,7 @@ swagger:
 
 pac4j:
   clients:
-    - org.pac4j.core.client.direct.AnonymousClient: {}          
-  bypassPaths:
-    - /api/info
-    - /api/server/info
-    - /api/server/platforms
-    - /api/auth/authorized      
+    - org.pac4j.core.client.direct.AnonymousClient: {}
 
 applicationName: Legend SDLC
 filterPriorities:

diff --git a/installers/omnibus/components/sdlc/config/config.gitlab-oauth.slim.yml b/installers/omnibus/components/sdlc/config/config.gitlab-oauth.slim.yml
@@ -80,4 +80,4 @@ gitLab:
   app:
     id: ${LEGEND_OMNIBUS_CONFIG_GITLAB_OAUTH_APPLICATION_ID}
     secret: ${LEGEND_OMNIBUS_CONFIG_GITLAB_OAUTH_APPLICATION_SECRET}
-    redirectURI: ${LEGEND_OMNIBUS_CONFIG_GITLAB_OAUTH_REDIRECT_BASE_URL}/sdlc/api/auth/callback
+    redirectURI: ${LEGEND_OMNIBUS_CONFIG_GITLAB_OAUTH_REDIRECT_BASE_URL}/api/auth/callback
diff --git a/installers/omnibus/components/sdlc/config/config.gitlab-pat.slim.yml b/installers/omnibus/components/sdlc/config/config.gitlab-pat.slim.yml
@@ -41,6 +41,9 @@ pac4j:
         apiVersion: v4
   bypassPaths:
     - /api/info
+    - /api/server/info
+    - /api/server/platforms
+    - /api/auth/authorized
 
 applicationName: Legend SDLC
 filterPriorities:

diff --git a/installers/omnibus/components/sdlc/config/config.in-memory.slim.yml b/installers/omnibus/components/sdlc/config/config.in-memory.slim.yml
@@ -32,12 +32,7 @@ swagger:
 
 pac4j:
   clients:
-    - org.pac4j.core.client.direct.AnonymousClient: {}          
-  bypassPaths:
-    - /api/info
-    - /api/server/info
-    - /api/server/platforms
-    - /api/auth/authorized      
+    - org.pac4j.core.client.direct.AnonymousClient: {}   
 
 applicationName: Legend SDLC
 filterPriorities:

diff --git a/installers/omnibus/components/sdlc/config/config.yml b/installers/omnibus/components/sdlc/config/config.yml
@@ -41,6 +41,9 @@ pac4j:
         apiVersion: v4
   bypassPaths:
     - /api/info
+    - /api/server/info
+    - /api/server/platforms
+    - /api/auth/authorized
 
 applicationName: Legend SDLC
 filterPriorities:

diff --git a/installers/omnibus/components/studio/config/server-config.yml b/installers/omnibus/components/studio/config/server-config.yml
@@ -1,10 +1,14 @@
 server:
-  type: simple
-  applicationContextPath: /
-  adminContextPath: /studio/admin
-  connector:
-    type: http
-    port: ${LEGEND_OMNIBUS_STUDIO_PORT}
+  applicationConnectors:
+    - type: http
+      port: ${LEGEND_OMNIBUS_STUDIO_PORT}
+      maxRequestHeaderSize: 128KiB
+  gzip:
+    includedMethods:
+      - GET
+      - POST
+  requestLog:
+    appenders: []
 sessionCookie: LEGEND_STUDIO_JSESSIONID
 logging:
   level: INFO
@@ -17,11 +21,8 @@ logging:
     - type: console
 
 pac4j:
-  callbackPrefix: /api/pac4j
   clients:
     - org.pac4j.core.client.direct.AnonymousClient: {}
-  bypassPaths:
-    - /studio/admin/healthcheck
 
 html5Router: true
 routerExemptPaths:

diff --git a/installers/omnibus/scripts/start.esg.sh b/installers/omnibus/scripts/start.esg.sh
@@ -13,7 +13,7 @@ YELLOW='\033[1;33m'
 NC='\033[0m' # No color
 # --------------------------------------------------------------------
 
-IMAGE=finos/legend-omnibus:2023-11-07-example-esg
+IMAGE=finos/legend-omnibus:latest-example-esg
 
 # ------------------------------- Check -------------------------------
 # Check if repulling image is necessary

diff --git a/installers/omnibus/scripts/start.example-esg.latest.sh b/installers/omnibus/scripts/start.example-esg.latest.sh
diff --git a/installers/omnibus/variants/example-esg/Dockerfile b/installers/omnibus/variants/example-esg/Dockerfile
@@ -9,7 +9,9 @@ ARG LEGEND_OMNIBUS_STUDIO_VERSION
 FROM finos/legend-engine-server:${LEGEND_OMNIBUS_ENGINE_VERSION} AS engine-image
 FROM finos/legend-sdlc-server:${LEGEND_OMNIBUS_SDLC_VERSION} AS sdlc-image
 FROM finos/legend-studio:${LEGEND_OMNIBUS_STUDIO_VERSION} AS studio-image
-FROM finos/legend-sdlc-server-fs:${LEGEND_OMNIBUS_SDLC_DEMO_FILE_SYSTEM_VERSION} AS sdlc-fs-image
+# TODO: to be removed when we make SDLC backend configurable
+# See https://github.com/finos/legend-sdlc/issues/722
+FROM finos/legend-sdlc-server-fs:${LEGEND_OMNIBUS_SDLC_DEMO_FILE_SYSTEM_VERSION} AS sdlc-file-system-backend-image
 
 FROM eclipse-temurin:11.0.19_7-jdk-jammy
 RUN apt-get update && apt-get install -y wget nano
@@ -39,10 +41,12 @@ ENV LEGEND_OMNIBUS_NGINX_PORT=${LEGEND_OMNIBUS_NGINX_PORT}
 # Legend Engine
 ARG LEGEND_OMNIBUS_ENGINE_VERSION
 ARG LEGEND_OMNIBUS_ENGINE_PORT
+ARG LEGEND_OMNIBUS_ENGINE_ADMIN_PORT
 ARG LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT
 
 ENV LEGEND_OMNIBUS_ENGINE_VERSION=${LEGEND_OMNIBUS_ENGINE_VERSION}
 ENV LEGEND_OMNIBUS_ENGINE_PORT=${LEGEND_OMNIBUS_ENGINE_PORT}
+ENV LEGEND_OMNIBUS_ENGINE_ADMIN_PORT=${LEGEND_OMNIBUS_ENGINE_ADMIN_PORT}
 ENV LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT=${LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT}
 
 # Legend SDLC
@@ -113,7 +117,7 @@ EXPOSE ${LEGEND_OMNIBUS_SDLC_PORT}
 # Copy over file-system backend libraries for Legend SDLC
 ARG SDLC_FS_DIR=/app/sdlc-fs
 RUN mkdir -p $SDLC_FS_DIR/lib
-COPY --from=sdlc-fs-image /app/bin/*.jar $SDLC_FS_DIR/lib
+COPY --from=sdlc-file-system-backend-image /app/bin/*.jar $SDLC_FS_DIR/lib
 RUN mkdir -p $LEGEND_OMNIBUS_SDLC_FILE_SYSTEM_ROOT_DIR
 RUN tar -xzf $SDLC_DIR/project-data__example-esg.tar.gz -C $LEGEND_OMNIBUS_SDLC_FILE_SYSTEM_ROOT_DIR
 RUN apt-get update && apt-get install -y git

diff --git a/installers/omnibus/variants/example-esg/build.sh b/installers/omnibus/variants/example-esg/build.sh
@@ -17,6 +17,7 @@ docker build \
   \
   --build-arg LEGEND_OMNIBUS_ENGINE_VERSION=$LEGEND_OMNIBUS_ENGINE_VERSION \
   --build-arg LEGEND_OMNIBUS_ENGINE_PORT=$LEGEND_OMNIBUS_ENGINE_PORT \
+  --build-arg LEGEND_OMNIBUS_ENGINE_ADMIN_PORT=$LEGEND_OMNIBUS_ENGINE_ADMIN_PORT \
   --build-arg LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT=$LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT \
   \
   --build-arg LEGEND_OMNIBUS_SDLC_VERSION=$LEGEND_OMNIBUS_SDLC_VERSION \

diff --git a/installers/omnibus/variants/example-esg/run.sh b/installers/omnibus/variants/example-esg/run.sh
@@ -11,6 +11,5 @@ docker run \
 	-p $LEGEND_OMNIBUS_ENGINE_PORT:$LEGEND_OMNIBUS_ENGINE_PORT \
 	-p $LEGEND_OMNIBUS_SDLC_PORT:$LEGEND_OMNIBUS_SDLC_PORT \
 	-p $LEGEND_OMNIBUS_STUDIO_PORT:$LEGEND_OMNIBUS_STUDIO_PORT \
-	-p 6910:5432 \
 	--env LEGEND_OMNIBUS_CONFIG_SDLC_MODE="file-system" \
 	legend-omnibus:latest-example-esg
diff --git a/installers/omnibus/variants/example-ghc-2023/Dockerfile b/installers/omnibus/variants/example-ghc-2023/Dockerfile
@@ -9,7 +9,9 @@ ARG LEGEND_OMNIBUS_STUDIO_VERSION
 FROM finos/legend-engine-server:${LEGEND_OMNIBUS_ENGINE_VERSION} AS engine-image
 FROM finos/legend-sdlc-server:${LEGEND_OMNIBUS_SDLC_VERSION} AS sdlc-image
 FROM finos/legend-studio:${LEGEND_OMNIBUS_STUDIO_VERSION} AS studio-image
-FROM finos/legend-sdlc-server-fs:${LEGEND_OMNIBUS_SDLC_DEMO_FILE_SYSTEM_VERSION} AS sdlc-fs-image
+# TODO: to be removed when we make SDLC backend configurable
+# See https://github.com/finos/legend-sdlc/issues/722
+FROM finos/legend-sdlc-server-fs:${LEGEND_OMNIBUS_SDLC_DEMO_FILE_SYSTEM_VERSION} AS sdlc-file-system-backend-image
 
 FROM eclipse-temurin:11.0.19_7-jdk-jammy
 RUN apt-get update && apt-get install -y wget nano
@@ -39,10 +41,12 @@ ENV LEGEND_OMNIBUS_NGINX_PORT=${LEGEND_OMNIBUS_NGINX_PORT}
 # Legend Engine
 ARG LEGEND_OMNIBUS_ENGINE_VERSION
 ARG LEGEND_OMNIBUS_ENGINE_PORT
+ARG LEGEND_OMNIBUS_ENGINE_ADMIN_PORT
 ARG LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT
 
 ENV LEGEND_OMNIBUS_ENGINE_VERSION=${LEGEND_OMNIBUS_ENGINE_VERSION}
 ENV LEGEND_OMNIBUS_ENGINE_PORT=${LEGEND_OMNIBUS_ENGINE_PORT}
+ENV LEGEND_OMNIBUS_ENGINE_ADMIN_PORT=${LEGEND_OMNIBUS_ENGINE_ADMIN_PORT}
 ENV LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT=${LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT}
 
 # Legend SDLC
@@ -111,9 +115,11 @@ RUN chmod +x $SDLC_DIR/run-sdlc.file-system.sh
 EXPOSE ${LEGEND_OMNIBUS_SDLC_PORT}
 
 # Copy over file-system backend libraries for Legend SDLC
+# TODO: to be removed when we make SDLC backend configurable
+# See https://github.com/finos/legend-sdlc/issues/722
 ARG SDLC_FS_DIR=/app/sdlc-fs
 RUN mkdir -p $SDLC_FS_DIR/lib
-COPY --from=sdlc-fs-image /app/bin/*.jar $SDLC_FS_DIR/lib
+COPY --from=sdlc-file-system-backend-image /app/bin/*.jar $SDLC_FS_DIR/lib
 RUN mkdir -p $LEGEND_OMNIBUS_SDLC_FILE_SYSTEM_ROOT_DIR
 RUN tar -xzf $SDLC_DIR/project-data__example-ghc-2023.tar.gz -C $LEGEND_OMNIBUS_SDLC_FILE_SYSTEM_ROOT_DIR
 RUN apt-get update && apt-get install -y git

diff --git a/installers/omnibus/variants/example-ghc-2023/build.sh b/installers/omnibus/variants/example-ghc-2023/build.sh
@@ -17,6 +17,7 @@ docker build \
   \
   --build-arg LEGEND_OMNIBUS_ENGINE_VERSION=$LEGEND_OMNIBUS_ENGINE_VERSION \
   --build-arg LEGEND_OMNIBUS_ENGINE_PORT=$LEGEND_OMNIBUS_ENGINE_PORT \
+  --build-arg LEGEND_OMNIBUS_ENGINE_ADMIN_PORT=$LEGEND_OMNIBUS_ENGINE_ADMIN_PORT \
   --build-arg LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT=$LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT \
   \
   --build-arg LEGEND_OMNIBUS_SDLC_VERSION=$LEGEND_OMNIBUS_SDLC_VERSION \

diff --git a/installers/omnibus/variants/full/Dockerfile b/installers/omnibus/variants/full/Dockerfile
@@ -50,10 +50,12 @@ ENV LEGEND_OMNIBUS_GITLAB_ROOT_PASSWORD=${LEGEND_OMNIBUS_GITLAB_ROOT_PASSWORD}
 # Legend Engine
 ARG LEGEND_OMNIBUS_ENGINE_VERSION
 ARG LEGEND_OMNIBUS_ENGINE_PORT
+ARG LEGEND_OMNIBUS_ENGINE_ADMIN_PORT
 ARG LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT
 
 ENV LEGEND_OMNIBUS_ENGINE_VERSION=${LEGEND_OMNIBUS_ENGINE_VERSION}
 ENV LEGEND_OMNIBUS_ENGINE_PORT=${LEGEND_OMNIBUS_ENGINE_PORT}
+ENV LEGEND_OMNIBUS_ENGINE_ADMIN_PORT=${LEGEND_OMNIBUS_ENGINE_ADMIN_PORT}
 ENV LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT=${LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT}
 
 # Legend SDLC

diff --git a/installers/omnibus/variants/full/build.sh b/installers/omnibus/variants/full/build.sh
@@ -22,6 +22,7 @@ docker build \
   \
   --build-arg LEGEND_OMNIBUS_ENGINE_VERSION=$LEGEND_OMNIBUS_ENGINE_VERSION \
   --build-arg LEGEND_OMNIBUS_ENGINE_PORT=$LEGEND_OMNIBUS_ENGINE_PORT \
+  --build-arg LEGEND_OMNIBUS_ENGINE_ADMIN_PORT=$LEGEND_OMNIBUS_ENGINE_ADMIN_PORT \
   --build-arg LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT=$LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT \
   \
   --build-arg LEGEND_OMNIBUS_SDLC_VERSION=$LEGEND_OMNIBUS_SDLC_VERSION \

diff --git a/installers/omnibus/variants/slim-dbs/Dockerfile b/installers/omnibus/variants/slim-dbs/Dockerfile
@@ -9,7 +9,9 @@ ARG LEGEND_OMNIBUS_STUDIO_VERSION
 FROM finos/legend-engine-server:${LEGEND_OMNIBUS_ENGINE_VERSION} AS engine-image
 FROM finos/legend-sdlc-server:${LEGEND_OMNIBUS_SDLC_VERSION} AS sdlc-image
 FROM finos/legend-studio:${LEGEND_OMNIBUS_STUDIO_VERSION} AS studio-image
-FROM legenddemo/legend-sdlc-server-demo:${LEGEND_OMNIBUS_SDLC_DEMO_IN_MEMORY_VERSION} as sdlc-demo-image
+# TODO: to be removed when we make SDLC backend configurable
+# See https://github.com/finos/legend-sdlc/issues/722
+FROM legenddemo/legend-sdlc-server-demo:${LEGEND_OMNIBUS_SDLC_DEMO_IN_MEMORY_VERSION} as sdlc-in-memory-backend-image
 
 FROM eclipse-temurin:11.0.19_7-jdk-jammy
 RUN apt-get update && apt-get install -y wget nano
@@ -39,10 +41,12 @@ ENV LEGEND_OMNIBUS_NGINX_PORT=${LEGEND_OMNIBUS_NGINX_PORT}
 # Legend Engine
 ARG LEGEND_OMNIBUS_ENGINE_VERSION
 ARG LEGEND_OMNIBUS_ENGINE_PORT
+ARG LEGEND_OMNIBUS_ENGINE_ADMIN_PORT
 ARG LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT
 
 ENV LEGEND_OMNIBUS_ENGINE_VERSION=${LEGEND_OMNIBUS_ENGINE_VERSION}
 ENV LEGEND_OMNIBUS_ENGINE_PORT=${LEGEND_OMNIBUS_ENGINE_PORT}
+ENV LEGEND_OMNIBUS_ENGINE_ADMIN_PORT=${LEGEND_OMNIBUS_ENGINE_ADMIN_PORT}
 ENV LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT=${LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT}
 
 # Legend SDLC
@@ -54,11 +58,6 @@ ENV LEGEND_OMNIBUS_SDLC_VERSION=${LEGEND_OMNIBUS_SDLC_VERSION}
 ENV LEGEND_OMNIBUS_SDLC_PORT=${LEGEND_OMNIBUS_SDLC_PORT}
 ENV LEGEND_OMNIBUS_SDLC_ADMIN_PORT=${LEGEND_OMNIBUS_SDLC_ADMIN_PORT}
 
-# Legend SDLC Demo
-ARG LEGEND_OMNIBUS_SDLC_DEMO_IN_MEMORY_VERSION
-
-ENV LEGEND_OMNIBUS_SDLC_DEMO_IN_MEMORY_VERSION=${LEGEND_OMNIBUS_SDLC_DEMO_IN_MEMORY_VERSION}
-
 # Legend Studio
 ARG LEGEND_OMNIBUS_STUDIO_VERSION
 ARG LEGEND_OMNIBUS_STUDIO_PORT
@@ -113,7 +112,7 @@ EXPOSE ${LEGEND_OMNIBUS_SDLC_PORT}
 ARG SDLC_DEMO_DIR=/app/sdlc-demo
 RUN mkdir -p $SDLC_DEMO_DIR/lib
 RUN mkdir -p $SDLC_DEMO_DIR/logs
-COPY --from=sdlc-demo-image /app/bin/*.jar $SDLC_DEMO_DIR/lib
+COPY --from=sdlc-in-memory-backend-image /app/bin/*.jar $SDLC_DEMO_DIR/lib
 
 # ---------------- Setup Legend Studio ----------------
 

diff --git a/installers/omnibus/variants/slim-dbs/build.sh b/installers/omnibus/variants/slim-dbs/build.sh
@@ -17,6 +17,7 @@ docker build \
   \
   --build-arg LEGEND_OMNIBUS_ENGINE_VERSION=$LEGEND_OMNIBUS_ENGINE_VERSION \
   --build-arg LEGEND_OMNIBUS_ENGINE_PORT=$LEGEND_OMNIBUS_ENGINE_PORT \
+  --build-arg LEGEND_OMNIBUS_ENGINE_ADMIN_PORT=$LEGEND_OMNIBUS_ENGINE_ADMIN_PORT \
   --build-arg LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT=$LEGEND_OMNIBUS_ENGINE_TEMP_DB_PORT \
   \
   --build-arg LEGEND_OMNIBUS_SDLC_VERSION=$LEGEND_OMNIBUS_SDLC_VERSION \