diff --git a/.helm/templates/celery-deployment.yaml b/.helm/templates/celery-deployment.yaml index 9aae80b9c..14daae8ad 100644 --- a/.helm/templates/celery-deployment.yaml +++ b/.helm/templates/celery-deployment.yaml @@ -54,6 +54,9 @@ spec: envFrom: - configMapRef: name: {{ .Chart.Name }}-base-config-map + env: + - name: JULIA_HOST + value: "localhost" volumeMounts: - name: {{ .Chart.Name }}-secrets-volume readOnly: true @@ -79,3 +82,22 @@ spec: limits: cpu: {{ .Values.celeryCpuLimit | quote }} memory: {{ .Values.celeryMemoryLimit | quote }} + + - name: {{ .Chart.Name }}-julia + image: {{ index .Values.werf.image "julia-api" }} + args: ["julia", "--project=/opt/julia_src", "http.jl"] + ports: + - containerPort: 8081 + env: + - name: JULIA_NUM_THREADS + value: "4" + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-base-config-map + resources: + requests: + cpu: {{ .Values.juliaCpuRequest | quote }} + memory: {{ .Values.juliaMemoryRequest | quote }} + limits: + cpu: {{ .Values.juliaCpuLimit | quote }} + memory: {{ .Values.juliaMemoryLimit | quote }} diff --git a/.helm/values.production.yaml b/.helm/values.production.yaml index 17667d9f9..026003ca6 100644 --- a/.helm/values.production.yaml +++ b/.helm/values.production.yaml @@ -6,11 +6,11 @@ djangoMemoryLimit: "2000Mi" celeryReplicas: 10 celeryMemoryRequest: "900Mi" celeryMemoryLimit: "900Mi" -juliaReplicas: 15 +juliaReplicas: 5 juliaCpuRequest: "2000m" juliaCpuLimit: "4000m" -juliaMemoryRequest: "16000Mi" -juliaMemoryLimit: "16000Mi" +juliaMemoryRequest: "12000Mi" +juliaMemoryLimit: "12000Mi" juliaDeploymentStrategy: rollingUpdate: maxSurge: "0%" diff --git a/.helm/values.staging.yaml b/.helm/values.staging.yaml index 78b346218..5bc76eb41 100644 --- a/.helm/values.staging.yaml +++ b/.helm/values.staging.yaml @@ -1,6 +1,4 @@ appEnv: staging djangoSettingsModule: reopt_api.staging_settings -juliaCpuRequest: "1000m" -juliaCpuLimit: "4000m" juliaMemoryRequest: "8000Mi" juliaMemoryLimit: "8000Mi" \ No newline at end of file diff --git a/.helm/values.yaml b/.helm/values.yaml index 5de3f46fb..2e227dfc3 100644 --- a/.helm/values.yaml +++ b/.helm/values.yaml @@ -9,7 +9,7 @@ juliaHost: "{{ .Chart.Name }}-julia-service" redisHost: "{{ .Chart.Name }}-redis-service" redisPort: 6379 djangoReplicas: 2 -djangoCpuRequest: "100m" +djangoCpuRequest: "2000m" djangoCpuLimit: "4000m" djangoMemoryRequest: "1600Mi" djangoMemoryLimit: "1600Mi" @@ -18,8 +18,8 @@ celeryCpuRequest: "100m" celeryCpuLimit: "2000m" celeryMemoryRequest: "700Mi" celeryMemoryLimit: "700Mi" -juliaReplicas: 2 -juliaCpuRequest: "1000m" +juliaReplicas: 1 +juliaCpuRequest: "2000m" juliaCpuLimit: "4000m" juliaMemoryRequest: "8000Mi" juliaMemoryLimit: "8000Mi" diff --git a/Jenkinsfile-restart-celery-julia b/Jenkinsfile-restart-celery-julia new file mode 100644 index 000000000..5665a6cb4 --- /dev/null +++ b/Jenkinsfile-restart-celery-julia @@ -0,0 +1,114 @@ +@Library("tada-jenkins-library") _ + +pipeline { + agent any + environment { + TZ = 'America/Denver' // MST/MDT (handles daylight saving automatically) + DEPLOY_IMAGE_REPO_DOMAIN = credentials("reopt-api-image-repo-domain") + DEVELOPMENT_BASE_DOMAIN = credentials("reopt-api-development-base-domain") + DEVELOPMENT_TEMP_BASE_DOMAIN = credentials("reopt-api-development-temp-base-domain") + STAGING_BASE_DOMAIN = credentials("reopt-api-staging-base-domain") + STAGING_TEMP_BASE_DOMAIN = credentials("reopt-api-staging-temp-base-domain") + PRODUCTION_DOMAIN = credentials("reopt-api-production-domain") + XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host") + NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root") + } + triggers { + cron('15 13 * * *') // 1:15 PM MST/MDT + } + + parameters { + booleanParam( + name: "DEVELOPMENT_DEPLOY", + defaultValue: false, + description: "Development Deploy: Deploy to development.", + ) + + booleanParam( + name: "STAGING_DEPLOY", + defaultValue: false, + description: "Staging Deploy: Deploy to staging for a non-master branch (master will always be deployed).", + ) + } + + stages { + stage("deploy-agent") { + agent { + docker { + image "${DEPLOY_IMAGE_REPO_DOMAIN}/tada-public/tada-jenkins-kube-deploy:werf-1.2" + args tadaDockerInDockerArgs() + } + } + + environment { + TMPDIR = tadaDockerInDockerTmp() + } + + stages { + stage('Rolling Restart Celery and Julia') { + stages { + stage("deploy-development") { + when { expression { params.DEVELOPMENT_DEPLOY } } + + environment { + DEPLOY_ENV = "development" + } + + steps { + withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { + tadaWithWerfEnv(rancherProject: "reopt-api-dev", primaryBranch: "master", dbBaseName: "reopt_api_development", baseDomain: "${DEVELOPMENT_BASE_DOMAIN}") { + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" + + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" + } + } + } + } + + stage("deploy-staging") { + when { expression { params.STAGING_DEPLOY || env.BRANCH_NAME == "master" } } + + environment { + DEPLOY_ENV = "staging" + } + + steps { + withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod4"]) { + tadaWithWerfEnv(rancherProject: "reopt-api-staging", primaryBranch: "master", dbBaseName: "reopt_api_staging", baseDomain: "${STAGING_BASE_DOMAIN}") { + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" + + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" + } + } + } + } + + stage("deploy-production") { + when { branch "master" } + + environment { + DEPLOY_ENV = "production" + } + + steps { + withKubeConfig([credentialsId: "kubeconfig-nrel-reopt-prod5"]) { + tadaWithWerfEnv(rancherProject: "reopt-api-production", primaryBranch: "master") { + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-celery-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-celery-deployment --timeout=10m" + + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout restart deployment/reopt-api-julia-deployment" + sh "kubectl -n '${env.DEPLOY_APP_NAMESPACE_NAME}' rollout status deployment/reopt-api-julia-deployment --timeout=10m" + } + } + } + } + } + } + } + } + } +} diff --git a/config/gunicorn.conf.py b/config/gunicorn.conf.py index 4d125349b..879818060 100644 --- a/config/gunicorn.conf.py +++ b/config/gunicorn.conf.py @@ -14,7 +14,7 @@ if os.environ.get('K8S_DEPLOY') is None: workers = multiprocessing.cpu_count() else: - workers = 8 + workers = 4 # Note that the app currently has threading issues, so we explicitly want a # non-thread worker process model. diff --git a/julia_src/http.jl b/julia_src/http.jl index cd12baa0e..2f94993fc 100644 --- a/julia_src/http.jl +++ b/julia_src/http.jl @@ -662,4 +662,4 @@ HTTP.register!(ROUTER, "GET", "/health", health) HTTP.register!(ROUTER, "GET", "/get_existing_chiller_default_cop", get_existing_chiller_default_cop) HTTP.register!(ROUTER, "GET", "/get_ashp_defaults", get_ashp_defaults) HTTP.register!(ROUTER, "GET", "/pv_cost_defaults", pv_cost_defaults) -HTTP.serve(ROUTER, "0.0.0.0", 8081, reuseaddr=true) +HTTP.serve(ROUTER, "0.0.0.0", 8081, reuseaddr=true, access_log=combined_logfmt)