Combined Celery+Julia Pods and Cron-job rolling restarts

.helm/templates/celery-deployment.yaml

@@ -54,6 +54,9 @@ spec:
           envFrom:
             - configMapRef:
                 name: {{ .Chart.Name }}-base-config-map
+          env:
+            - name: JULIA_HOST
+              value: "localhost"
           volumeMounts:
             - name: {{ .Chart.Name }}-secrets-volume
               readOnly: true
@@ -79,3 +82,22 @@ spec:
             limits:
               cpu: {{ .Values.celeryCpuLimit | quote }}
               memory: {{ .Values.celeryMemoryLimit | quote }}
+
+        - name: {{ .Chart.Name }}-julia
+          image: {{ index .Values.werf.image "julia-api" }}
+          args: ["julia", "--project=/opt/julia_src", "http.jl"]
+          ports:
+            - containerPort: 8081
+          env:
+            - name: JULIA_NUM_THREADS
+              value: "4"
+          envFrom:
+            - configMapRef:
+                name: {{ .Chart.Name }}-base-config-map
+          resources:
+            requests:
+              cpu:  {{ .Values.juliaCpuRequest | quote }}
+              memory: {{ .Values.juliaMemoryRequest | quote }}
+            limits:
+              cpu: {{ .Values.juliaCpuLimit | quote }}
+              memory: {{ .Values.juliaMemoryLimit | quote }}

.helm/values.production.yaml

@@ -6,11 +6,11 @@ djangoMemoryLimit: "2000Mi"
 celeryReplicas: 10
 celeryMemoryRequest: "900Mi"
 celeryMemoryLimit: "900Mi"
-juliaReplicas: 15
+juliaReplicas: 5
 juliaCpuRequest: "2000m"
 juliaCpuLimit: "4000m"
-juliaMemoryRequest: "16000Mi"
-juliaMemoryLimit: "16000Mi"
+juliaMemoryRequest: "12000Mi"
+juliaMemoryLimit: "12000Mi"
 juliaDeploymentStrategy:
   rollingUpdate:
     maxSurge: "0%"

.helm/values.staging.yaml

@@ -1,6 +1,4 @@
 appEnv: staging
 djangoSettingsModule: reopt_api.staging_settings
-juliaCpuRequest: "1000m"
-juliaCpuLimit: "4000m"
 juliaMemoryRequest: "8000Mi"
 juliaMemoryLimit: "8000Mi"

.helm/values.yaml

@@ -9,7 +9,7 @@ juliaHost: "{{ .Chart.Name }}-julia-service"
 redisHost: "{{ .Chart.Name }}-redis-service"
 redisPort: 6379
 djangoReplicas: 2
-djangoCpuRequest: "100m"
+djangoCpuRequest: "2000m"
 djangoCpuLimit: "4000m"
 djangoMemoryRequest: "1600Mi"
 djangoMemoryLimit: "1600Mi"
@@ -18,8 +18,8 @@ celeryCpuRequest: "100m"
 celeryCpuLimit: "2000m"
 celeryMemoryRequest: "700Mi"
 celeryMemoryLimit: "700Mi"
-juliaReplicas: 2
-juliaCpuRequest: "1000m"
+juliaReplicas: 1
+juliaCpuRequest: "2000m"
 juliaCpuLimit: "4000m"
 juliaMemoryRequest: "8000Mi"
 juliaMemoryLimit: "8000Mi"

Jenkinsfile-restart-celery-julia

@@ -0,0 +1,114 @@
+@Library("tada-jenkins-library") _
+
+pipeline {
+  agent any
+  environment {
+    TZ = 'America/Denver' // MST/MDT (handles daylight saving automatically)
+    DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain")
+    DEVELOPMENT_BASE_DOMAIN = credentials("reopt-api-development-base-domain")
+    DEVELOPMENT_TEMP_BASE_DOMAIN = credentials("reopt-api-development-temp-base-domain")
+    STAGING_BASE_DOMAIN = credentials("reopt-api-staging-base-domain")
+    STAGING_TEMP_BASE_DOMAIN = credentials("reopt-api-staging-temp-base-domain")
+    PRODUCTION_DOMAIN = credentials("reopt-api-production-domain")
+    XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host")
+    NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root")
+  }
+  triggers {
+    cron('15 13 * * *') // 1:15 PM MST/MDT
+  }
+
+  parameters {
+    booleanParam(
+      name: "DEVELOPMENT_DEPLOY",
+      defaultValue: false,
+      description: "Development Deploy: Deploy to development.",
+    )
+
+    booleanParam(
+      name: "STAGING_DEPLOY",
+      defaultValue: false,
+      description: "Staging Deploy: Deploy to staging for a non-master branch (master will always be deployed).",
+    )
+  }
+
+  stages {
+    stage("deploy-agent") {
+      agent {
+        docker {
+          image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2"
+          args tadaDockerInDockerArgs()
+        }
+      }
+
+      environment {
+        TMPDIR = tadaDockerInDockerTmp()
+      }
+
+      stages {
+        stage('Rolling Restart Celery and Julia') {
+          stages {
+            stage("deploy-development") {
+              when { expression { params.DEVELOPMENT_DEPLOY } }
+
+              environment {
+                DEPLOY_ENV = "development"
+              }
+
+              steps {
+                withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) {
+                  tadaWithWerfEnv(rancherProject: "reopt-api-dev", primaryBranch: "master", dbBaseName: "reopt_api_development", baseDomain: "${DEVELOPMENT_BASE_DOMAIN}") {
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment"
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m"
+
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment"
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m"
+                  }
+                }
+              }
+            }
+
+            stage("deploy-staging") {
+              when { expression { params.STAGING_DEPLOY || env.BRANCH_NAME == "master" } }
+
+              environment {
+                DEPLOY_ENV = "staging"
+              }
+
+              steps {
+                withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) {
+                  tadaWithWerfEnv(rancherProject: "reopt-api-staging", primaryBranch: "master", dbBaseName: "reopt_api_staging", baseDomain: "${STAGING_BASE_DOMAIN}") {
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment"
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m"
+
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment"
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m"
+                  }
+                }
+              }
+            }
+
+            stage("deploy-production") {
+              when { branch "master" }
+
+              environment {
+                DEPLOY_ENV = "production"
+              }
+
+              steps {
+                withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod5"]) {
+                  tadaWithWerfEnv(rancherProject: "reopt-api-production", primaryBranch: "master") {
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment"
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m"
+
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment"
+                    sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

config/gunicorn.conf.py

@@ -14,7 +14,7 @@
 if os.environ.get('K8S_DEPLOY') is None:
     workers = multiprocessing.cpu_count()
 else:
-    workers = 8
+    workers = 4
 
 # Note that the app currently has threading issues, so we explicitly want a
 # non-thread worker process model.

julia_src/http.jl

@@ -662,4 +662,4 @@ HTTP.register!(ROUTER, "GET", "/health", health)
 HTTP.register!(ROUTER, "GET", "/get_existing_chiller_default_cop", get_existing_chiller_default_cop)
 HTTP.register!(ROUTER, "GET", "/get_ashp_defaults", get_ashp_defaults)
 HTTP.register!(ROUTER, "GET", "/pv_cost_defaults", pv_cost_defaults)
-HTTP.serve(ROUTER, "0.0.0.0", 8081, reuseaddr=true)
+HTTP.serve(ROUTER, "0.0.0.0", 8081, reuseaddr=true, access_log=combined_logfmt)