From 0a21ca146f512364217f2dd4ad6f561ee54cd21d Mon Sep 17 00:00:00 2001 From: bill-becker Date: Sat, 23 Aug 2025 09:57:15 -0600 Subject: [PATCH 01/12] Add Jenkins pipeline for rolling restart of julia/celery containers --- Jenkinsfile-restart-celery-julia | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 Jenkinsfile-restart-celery-julia diff --git a/Jenkinsfile-restart-celery-julia b/Jenkinsfile-restart-celery-julia new file mode 100644 index 000000000..6999c49dc --- /dev/null +++ b/Jenkinsfile-restart-celery-julia @@ -0,0 +1,18 @@ +pipeline { + agent any + environment { + TZ = 'America/Denver' // MST/MDT (handles daylight saving automatically) + CHART_NAME = 'reopt-api-staging' // Set your chart name here + } + triggers { + cron('15 10 * * *') // 10:15 AM MST/MDT + } + stages { + stage('Rolling Restart Celery and Julia') { + steps { + sh "kubectl rollout restart deployment/${CHART_NAME}-celery-deployment" + sh "kubectl rollout restart deployment/${CHART_NAME}-julia-deployment" + } + } + } +} \ No newline at end of file From 1d7ce9e4c150e89d9ff787fdcc23636a1ec4a037 Mon Sep 17 00:00:00 2001 From: bill-becker Date: Sat, 23 Aug 2025 13:08:22 -0600 Subject: [PATCH 02/12] Add tada-jenkins-library to new Jenkinsfile pipeline file --- Jenkinsfile-restart-celery-julia | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile-restart-celery-julia b/Jenkinsfile-restart-celery-julia index 6999c49dc..942802143 100644 --- a/Jenkinsfile-restart-celery-julia +++ b/Jenkinsfile-restart-celery-julia @@ -1,3 +1,5 @@ +@Library("tada-jenkins-library") _ + pipeline { agent any environment { From f2a40adcd276f96e2ad0779cc6e84eb2a8fcdee3 Mon Sep 17 00:00:00 2001 From: bill-becker Date: Sat, 23 Aug 2025 13:09:58 -0600 Subject: [PATCH 03/12] Update cron time for restart-celery-julia --- Jenkinsfile-restart-celery-julia | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile-restart-celery-julia b/Jenkinsfile-restart-celery-julia index 942802143..9df0d1b9c 100644 --- a/Jenkinsfile-restart-celery-julia +++ b/Jenkinsfile-restart-celery-julia @@ -7,7 +7,7 @@ pipeline { CHART_NAME = 'reopt-api-staging' // Set your chart name here } triggers { - cron('15 10 * * *') // 10:15 AM MST/MDT + cron('15 13 * * *') // 1:15 PM MST/MDT } stages { stage('Rolling Restart Celery and Julia') { From 8ea07561ee5bd5b147390216ddd4839302c005bd Mon Sep 17 00:00:00 2001 From: bill-becker Date: Sat, 23 Aug 2025 13:20:28 -0600 Subject: [PATCH 04/12] Add Julia container to the Celery pod for linking celery-to-julia for /job runs --- .helm/templates/celery-deployment.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.helm/templates/celery-deployment.yaml b/.helm/templates/celery-deployment.yaml index 9aae80b9c..264ce4a46 100644 --- a/.helm/templates/celery-deployment.yaml +++ b/.helm/templates/celery-deployment.yaml @@ -79,3 +79,22 @@ spec: limits: cpu: {{ .Values.celeryCpuLimit | quote }} memory: {{ .Values.celeryMemoryLimit | quote }} + + - name: {{ .Chart.Name }}-julia + image: {{ index .Values.werf.image "julia-api" }} + args: ["julia", "--project=/opt/julia_src", "http.jl"] + ports: + - containerPort: 8081 + env: + - name: JULIA_NUM_THREADS + value: "4" + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-base-config-map + resources: + requests: + cpu: {{ .Values.juliaCpuRequest | quote }} + memory: {{ .Values.juliaMemoryRequest | quote }} + limits: + cpu: {{ .Values.juliaCpuLimit | quote }} + memory: {{ .Values.juliaMemoryLimit | quote }} From 9a8f7f1753856577a1bf8f0b50e97216f2129da8 Mon Sep 17 00:00:00 2001 From: bill-becker Date: Sat, 23 Aug 2025 13:21:56 -0600 Subject: [PATCH 05/12] Update deploy resource values and reduce django workers from 8 to 4 with gunicorn --- .helm/values.production.yaml | 6 +++--- .helm/values.staging.yaml | 2 -- .helm/values.yaml | 6 +++--- config/gunicorn.conf.py | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.helm/values.production.yaml b/.helm/values.production.yaml index 17667d9f9..026003ca6 100644 --- a/.helm/values.production.yaml +++ b/.helm/values.production.yaml @@ -6,11 +6,11 @@ djangoMemoryLimit: "2000Mi" celeryReplicas: 10 celeryMemoryRequest: "900Mi" celeryMemoryLimit: "900Mi" -juliaReplicas: 15 +juliaReplicas: 5 juliaCpuRequest: "2000m" juliaCpuLimit: "4000m" -juliaMemoryRequest: "16000Mi" -juliaMemoryLimit: "16000Mi" +juliaMemoryRequest: "12000Mi" +juliaMemoryLimit: "12000Mi" juliaDeploymentStrategy: rollingUpdate: maxSurge: "0%" diff --git a/.helm/values.staging.yaml b/.helm/values.staging.yaml index 78b346218..5bc76eb41 100644 --- a/.helm/values.staging.yaml +++ b/.helm/values.staging.yaml @@ -1,6 +1,4 @@ appEnv: staging djangoSettingsModule: reopt_api.staging_settings -juliaCpuRequest: "1000m" -juliaCpuLimit: "4000m" juliaMemoryRequest: "8000Mi" juliaMemoryLimit: "8000Mi" \ No newline at end of file diff --git a/.helm/values.yaml b/.helm/values.yaml index 5de3f46fb..2e227dfc3 100644 --- a/.helm/values.yaml +++ b/.helm/values.yaml @@ -9,7 +9,7 @@ juliaHost: "{{ .Chart.Name }}-julia-service" redisHost: "{{ .Chart.Name }}-redis-service" redisPort: 6379 djangoReplicas: 2 -djangoCpuRequest: "100m" +djangoCpuRequest: "2000m" djangoCpuLimit: "4000m" djangoMemoryRequest: "1600Mi" djangoMemoryLimit: "1600Mi" @@ -18,8 +18,8 @@ celeryCpuRequest: "100m" celeryCpuLimit: "2000m" celeryMemoryRequest: "700Mi" celeryMemoryLimit: "700Mi" -juliaReplicas: 2 -juliaCpuRequest: "1000m" +juliaReplicas: 1 +juliaCpuRequest: "2000m" juliaCpuLimit: "4000m" juliaMemoryRequest: "8000Mi" juliaMemoryLimit: "8000Mi" diff --git a/config/gunicorn.conf.py b/config/gunicorn.conf.py index 4d125349b..879818060 100644 --- a/config/gunicorn.conf.py +++ b/config/gunicorn.conf.py @@ -14,7 +14,7 @@ if os.environ.get('K8S_DEPLOY') is None: workers = multiprocessing.cpu_count() else: - workers = 8 + workers = 4 # Note that the app currently has threading issues, so we explicitly want a # non-thread worker process model. From d30a32c0df9a5926857ee19ec0894926985d1a19 Mon Sep 17 00:00:00 2001 From: bill-becker Date: Sat, 23 Aug 2025 13:52:54 -0600 Subject: [PATCH 06/12] Update celery pod to use localhost for juliaHost --- .helm/templates/celery-deployment.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.helm/templates/celery-deployment.yaml b/.helm/templates/celery-deployment.yaml index 264ce4a46..998946ba9 100644 --- a/.helm/templates/celery-deployment.yaml +++ b/.helm/templates/celery-deployment.yaml @@ -54,6 +54,9 @@ spec: envFrom: - configMapRef: name: {{ .Chart.Name }}-base-config-map + env: + - name: juliaHost + value: "localhost" volumeMounts: - name: {{ .Chart.Name }}-secrets-volume readOnly: true From bcf97262d661bd855cc360c397d3706a8298ecf3 Mon Sep 17 00:00:00 2001 From: bill-becker Date: Sat, 23 Aug 2025 14:09:01 -0600 Subject: [PATCH 07/12] Fix JULIA_HOST name for local celery to Julia comm --- .helm/templates/celery-deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.helm/templates/celery-deployment.yaml b/.helm/templates/celery-deployment.yaml index 998946ba9..8c3c08939 100644 --- a/.helm/templates/celery-deployment.yaml +++ b/.helm/templates/celery-deployment.yaml @@ -55,8 +55,8 @@ spec: - configMapRef: name: {{ .Chart.Name }}-base-config-map env: - - name: juliaHost - value: "localhost" + - name: JULIA_HOST + value: "http://localhost:8081" volumeMounts: - name: {{ .Chart.Name }}-secrets-volume readOnly: true From bae2a77ab17dacfcf3475333d5533f80f4df4141 Mon Sep 17 00:00:00 2001 From: bill-becker Date: Sat, 23 Aug 2025 14:17:01 -0600 Subject: [PATCH 08/12] Fix indention of the celery container env section --- .helm/templates/celery-deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.helm/templates/celery-deployment.yaml b/.helm/templates/celery-deployment.yaml index 8c3c08939..0ab927214 100644 --- a/.helm/templates/celery-deployment.yaml +++ b/.helm/templates/celery-deployment.yaml @@ -54,9 +54,9 @@ spec: envFrom: - configMapRef: name: {{ .Chart.Name }}-base-config-map - env: - - name: JULIA_HOST - value: "http://localhost:8081" + env: + - name: JULIA_HOST + value: "http://localhost:8081" volumeMounts: - name: {{ .Chart.Name }}-secrets-volume readOnly: true From 83d0b3e2335407fa04d0d4ce716858872e435870 Mon Sep 17 00:00:00 2001 From: bill-becker Date: Sat, 23 Aug 2025 14:28:07 -0600 Subject: [PATCH 09/12] Change JULIA_HOST back to just "localhost" --- .helm/templates/celery-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.helm/templates/celery-deployment.yaml b/.helm/templates/celery-deployment.yaml index 0ab927214..14daae8ad 100644 --- a/.helm/templates/celery-deployment.yaml +++ b/.helm/templates/celery-deployment.yaml @@ -56,7 +56,7 @@ spec: name: {{ .Chart.Name }}-base-config-map env: - name: JULIA_HOST - value: "http://localhost:8081" + value: "localhost" volumeMounts: - name: {{ .Chart.Name }}-secrets-volume readOnly: true From cd86fa62344b4f22675a23fbb183ea33032aa13f Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Sat, 30 Aug 2025 10:45:04 -0500 Subject: [PATCH 10/12] Add HTTP access logging to Julia's HTTP server. This will log requests hitting the Julia HTTP server making it a little more obvious what's happening in the logs. --- julia_src/http.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/julia_src/http.jl b/julia_src/http.jl index cd12baa0e..2f94993fc 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -662,4 +662,4 @@ HTTP.register!(ROUTER, "GET", "/health", health) HTTP.register!(ROUTER, "GET", "/get_existing_chiller_default_cop", get_existing_chiller_default_cop) HTTP.register!(ROUTER, "GET", "/get_ashp_defaults", get_ashp_defaults) HTTP.register!(ROUTER, "GET", "/pv_cost_defaults", pv_cost_defaults) -HTTP.serve(ROUTER, "0.0.0.0", 8081, reuseaddr=true) +HTTP.serve(ROUTER, "0.0.0.0", 8081, reuseaddr=true, access_log=combined_logfmt) From 3d3998e6f1fcef5ca41f7a91693b14feccd80105 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Sat, 30 Aug 2025 11:33:37 -0500 Subject: [PATCH 11/12] Attempt to setup Jenkinsfile to restart deployments --- Jenkinsfile-restart-celery-julia | 92 +++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile-restart-celery-julia b/Jenkinsfile-restart-celery-julia index 9df0d1b9c..237aad9ae 100644 --- a/Jenkinsfile-restart-celery-julia +++ b/Jenkinsfile-restart-celery-julia @@ -4,17 +4,97 @@ pipeline { agent any environment { TZ = 'America/Denver' // MST/MDT (handles daylight saving automatically) - CHART_NAME = 'reopt-api-staging' // Set your chart name here + DEVELOPMENT_BASE_DOMAIN = credentials("reopt-api-development-base-domain") + STAGING_BASE_DOMAIN = credentials("reopt-api-staging-base-domain") + PRODUCTION_DOMAIN = credentials("reopt-api-production-domain") } triggers { cron('15 13 * * *') // 1:15 PM MST/MDT } + + parameters { + booleanParam( + name: "DEVELOPMENT_DEPLOY", + defaultValue: false, + description: "Development Deploy: Deploy to development.", + ) + + booleanParam( + name: "STAGING_DEPLOY", + defaultValue: false, + description: "Staging Deploy: Deploy to staging for a non-master branch (master will always be deployed).", + ) + } + stages { - stage('Rolling Restart Celery and Julia') { - steps { - sh "kubectl rollout restart deployment/${CHART_NAME}-celery-deployment" - sh "kubectl rollout restart deployment/${CHART_NAME}-julia-deployment" + stage("deploy-agent") { + agent { + docker { + image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" + args tadaDockerInDockerArgs() + } + } + + environment { + TMPDIR = tadaDockerInDockerTmp() + } + + stages { + stage('Rolling Restart Celery and Julia') { + stages { + stage("deploy-development") { + when { expression { params.DEVELOPMENT_DEPLOY } } + + environment { + DEPLOY_ENV = "development" + } + + steps { + withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { + tadaWithWerfEnv(rancherProject: "reopt-api-dev", primaryBranch: "master", dbBaseName: "reopt_api_development", baseDomain: "${DEVELOPMENT_BASE_DOMAIN}") { + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" + } + } + } + } + + stage("deploy-staging") { + when { expression { params.STAGING_DEPLOY || env.BRANCH_NAME == "master" } } + + environment { + DEPLOY_ENV = "staging" + } + + steps { + withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { + tadaWithWerfEnv(rancherProject: "reopt-api-staging", primaryBranch: "master", dbBaseName: "reopt_api_staging", baseDomain: "${STAGING_BASE_DOMAIN}") { + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" + } + } + } + } + + stage("deploy-production") { + when { branch "master" } + + environment { + DEPLOY_ENV = "production" + } + + steps { + withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod5"]) { + tadaWithWerfEnv(rancherProject: "reopt-api-production", primaryBranch: "master") { + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" + } + } + } + } + } + } } } } -} \ No newline at end of file +} From e161e6d1372acca5d6d12165651a6ce2b3b5ebb1 Mon Sep 17 00:00:00 2001 From: Nick Muerdter <12112+GUI@users.noreply.github.com> Date: Sat, 30 Aug 2025 11:49:54 -0500 Subject: [PATCH 12/12] Fixes for restart task - Add some missing variables needed even for this basic restart task. - Wait for rollout restarts to complete so we know if they've been successful or not. --- Jenkinsfile-restart-celery-julia | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Jenkinsfile-restart-celery-julia b/Jenkinsfile-restart-celery-julia index 237aad9ae..5665a6cb4 100644 --- a/Jenkinsfile-restart-celery-julia +++ b/Jenkinsfile-restart-celery-julia @@ -4,9 +4,14 @@ pipeline { agent any environment { TZ = 'America/Denver' // MST/MDT (handles daylight saving automatically) + DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") DEVELOPMENT_BASE_DOMAIN = credentials("reopt-api-development-base-domain") + DEVELOPMENT_TEMP_BASE_DOMAIN = credentials("reopt-api-development-temp-base-domain") STAGING_BASE_DOMAIN = credentials("reopt-api-staging-base-domain") + STAGING_TEMP_BASE_DOMAIN = credentials("reopt-api-staging-temp-base-domain") PRODUCTION_DOMAIN = credentials("reopt-api-production-domain") + XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") + NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") } triggers { cron('15 13 * * *') // 1:15 PM MST/MDT @@ -53,7 +58,10 @@ pipeline { withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { tadaWithWerfEnv(rancherProject: "reopt-api-dev", primaryBranch: "master", dbBaseName: "reopt_api_development", baseDomain: "${DEVELOPMENT_BASE_DOMAIN}") { sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" } } } @@ -70,7 +78,10 @@ pipeline { withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { tadaWithWerfEnv(rancherProject: "reopt-api-staging", primaryBranch: "master", dbBaseName: "reopt_api_staging", baseDomain: "${STAGING_BASE_DOMAIN}") { sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" } } } @@ -87,7 +98,10 @@ pipeline { withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod5"]) { tadaWithWerfEnv(rancherProject: "reopt-api-production", primaryBranch: "master") { sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" } } }