Make Batch Job monitoring URL configurable from env var (#9)

.env.sample

@@ -5,49 +5,50 @@ DATABASE_PASSWORD=merlin
 DATABASE_NAME=merlin
 
 SENTRY_ENABLED=true
-SENTRY_DSN=[*** PASTE SENTRY DSN ****]
+# SENTRY_DSN=
 
 REACT_APP_HOMEPAGE=/merlin
-REACT_APP_MERLIN_API=http://localhost:8080/v1
+REACT_APP_MERLIN_API=http://localhost:3000/v1
 REACT_APP_OAUTH_CLIENT_ID=
-REACT_APP_ALERT_ENABLED=false
-REACT_APP_MONITORING_DASHBOARD_ENABLED=false
-REACT_APP_MLP_API=http://mlp.dev/api/v1
+REACT_APP_MLP_API=http://localhost:3001/v1
 
-NEWRELIC_ENABLED=true
+NEWRELIC_ENABLED=false
 NEWRELIC_APPNAME=merlin-api-dev
 NEWRELIC_IGNORESTATUSCODES=400,404
 
-IMG_BUILDER_CLUSTER_NAME=[*** PASTE k8s CLUSTER NAME ****]
-IMG_BUILDER_BUILD_CONTEXT_URI=[*** PASTE BUILD CONTEXT URI ****]
-IMG_BUILDER_PREDICTION_JOB_BUILD_CONTEXT_URI=[*** PASTE BUILD CONTEXT URI ****]
+# IMG_BUILDER_CLUSTER_NAME=
+# TODO: Set with GitHub path
+# IMG_BUILDER_BUILD_CONTEXT_URI=
+# IMG_BUILDER_PREDICTION_JOB_BUILD_CONTEXT_URI=
 IMG_BUILDER_PREDICTION_JOB_DOCKERFILE_PATH=docker/app.Dockerfile
 IMG_BUILDER_PREDICTION_JOB_BASE_IMAGE=ghcr.io/gojek/merlin-pyspark-base:v0.6.0
 IMG_BUILDER_NAMESPACE=mlp
 IMG_BUILDER_DOCKER_REGISTRY=ghcr.io/gojek
 IMG_BUILDER_TIMEOUT=10m
-IMG_BUILDER_GCP_PROJECT=[*** GCP PROJECT ****]
+# IMG_BUILDER_GCP_PROJECT=
 IMG_BUILDER_PREDICTION_JOB_CONTEXT_SUB_PATH="python/batch-predictor"
 IMG_BUILDER_CONTEXT_SUB_PATH="python/pyfunc-server"
 
-MODEL_CLUSTER_NAME=[*** PASTE k8s CLUSTER NAME FOR MODEL ****]
+# MODEL_CLUSTER_NAME=
 MODEL_DEPLOYMENT_TIMEOUT=10m
 
-VAULT_ADDRESS=[*** PASTE VAULT ADDRESS ****]
-VAULT_TOKEN=[*** PASTE VAULT TOKEN ****]
+# VAULT_ADDRESS=
+# VAULT_TOKEN=
 
 AUTHORIZATION_ENABLED=false
-ALERT_ENABLED=true
+ALERT_ENABLED=false
 MONITORING_DASHBOARD_ENABLED=true
+MONITORING_DASHBOARD_BASE_URL=http://monitoring.dev/model-dashboard
+MONITORING_DASHBOARD_JOB_BASE_URL=http://monitoring.dev/batch-job-dashboard
 
-GITLAB_BASE_URL=[*** PASTE GITLAB CONTEXT URI ****]
-GITLAB_TOKEN=[*** PASTE GITLAB TOKEN ****]
-GITLAB_DASHBOARD_REPOSITORY=[*** PASTE DASHBOARD REPOSITORY ****]
-GITLAB_DASHBOARD_BRANCH=[*** PASTE DASHBOARD BRANCH ****]
-GITLAB_ALERT_REPOSITORY=[*** PASTE ALERT REPOSITORY ****]
-GITLAB_ALERT_BRANCH=[*** PASTE ALERT BRANCH ****]
+# GITLAB_BASE_URL=
+# GITLAB_TOKEN=
+# GITLAB_DASHBOARD_REPOSITORY=
+# GITLAB_DASHBOARD_BRANCH=
+# GITLAB_ALERT_REPOSITORY=
+# GITLAB_ALERT_BRANCH=
 
-WARDEN_API_HOST=[*** PASTE HOST ****]
+# WARDEN_API_HOST=
 
-MLP_API_HOST=[*** PASTE HOST ****]
-MLP_API_ENCRYPTION_KEY=[*** PASTE ENCRYPTION KEY ****]
+# MLP_API_HOST=
+MLP_API_ENCRYPTION_KEY=password

api/cmd/main.go

@@ -195,15 +195,18 @@ func main() {
 
 	reactConfig := cfg.ReactAppConfig
 	uiEnv := uiEnvHandler{
-		OauthClientID:     reactConfig.OauthClientID,
-		Environment:       reactConfig.Environment,
-		SentryDSN:         reactConfig.SentryDSN,
-		DocURL:            reactConfig.DocURL,
-		AlertEnabled:      reactConfig.AlertEnabled,
-		MonitoringEnabled: reactConfig.MonitoringEnabled,
-		HomePage:          reactConfig.HomePage,
-		MerlinURL:         reactConfig.MerlinURL,
-		MlpURL:            reactConfig.MlpURL,
+		OauthClientID: reactConfig.OauthClientID,
+		Environment:   reactConfig.Environment,
+		SentryDSN:     reactConfig.SentryDSN,
+		DocURL:        reactConfig.DocURL,
+		HomePage:      reactConfig.HomePage,
+		MerlinURL:     reactConfig.MerlinURL,
+		MlpURL:        reactConfig.MlpURL,
+
+		MonitoringEnabled:              cfg.FeatureToggleConfig.MonitoringConfig.MonitoringEnabled,
+		MonitoringPredictionJobBaseURL: cfg.FeatureToggleConfig.MonitoringConfig.MonitoringJobBaseURL,
+
+		AlertEnabled: cfg.FeatureToggleConfig.AlertConfig.AlertEnabled,
 	}
 
 	uiHomePage := reactConfig.HomePage
@@ -220,15 +223,18 @@ func main() {
 }
 
 type uiEnvHandler struct {
-	OauthClientID     string `json:"REACT_APP_OAUTH_CLIENT_ID,omitempty"`
-	Environment       string `json:"REACT_APP_ENVIRONMENT,omitempty"`
-	SentryDSN         string `json:"REACT_APP_SENTRY_DSN,omitempty"`
-	DocURL            string `json:"REACT_APP_MERLIN_DOCS_URL,omitempty"`
-	AlertEnabled      bool   `json:"REACT_APP_ALERT_ENABLED"`
-	MonitoringEnabled bool   `json:"REACT_APP_MONITORING_DASHBOARD_ENABLED"`
-	HomePage          string `json:"REACT_APP_HOMEPAGE,omitempty"`
-	MerlinURL         string `json:"REACT_APP_MERLIN_API,omitempty"`
-	MlpURL            string `json:"REACT_APP_MLP_API,omitempty"`
+	OauthClientID string `json:"REACT_APP_OAUTH_CLIENT_ID,omitempty"`
+	Environment   string `json:"REACT_APP_ENVIRONMENT,omitempty"`
+	SentryDSN     string `json:"REACT_APP_SENTRY_DSN,omitempty"`
+	DocURL        string `json:"REACT_APP_MERLIN_DOCS_URL,omitempty"`
+	HomePage      string `json:"REACT_APP_HOMEPAGE,omitempty"`
+	MerlinURL     string `json:"REACT_APP_MERLIN_API,omitempty"`
+	MlpURL        string `json:"REACT_APP_MLP_API,omitempty"`
+
+	MonitoringEnabled              bool   `json:"REACT_APP_MONITORING_DASHBOARD_ENABLED"`
+	MonitoringPredictionJobBaseURL string `json:"REACT_APP_MONITORING_DASHBOARD_JOB_BASE_URL"`
+
+	AlertEnabled bool `json:"REACT_APP_ALERT_ENABLED"`
 }
 
 func (h uiEnvHandler) handler(w http.ResponseWriter, r *http.Request) {

api/config/config.go

@@ -108,12 +108,13 @@ type FeatureToggleConfig struct {
 }
 
 type MonitoringConfig struct {
-	MonitoringEnabled bool   `envconfig:"MONITORING_DASHBOARD_ENABLED" default:"true"`
-	MonitoringBaseURL string `envconfig:"MONITORING_DASHBOARD_BASE_URL"`
+	MonitoringEnabled    bool   `envconfig:"MONITORING_DASHBOARD_ENABLED" default:"false"`
+	MonitoringBaseURL    string `envconfig:"MONITORING_DASHBOARD_BASE_URL"`
+	MonitoringJobBaseURL string `envconfig:"MONITORING_DASHBOARD_JOB_BASE_URL"`
 }
 
 type AlertConfig struct {
-	AlertEnabled bool `envconfig:"ALERT_ENABLED" default:"true"`
+	AlertEnabled bool `envconfig:"ALERT_ENABLED" default:"false"`
 	GitlabConfig GitlabConfig
 	WardenConfig WardenConfig
 }

api/models/version_endpoint.go

@@ -73,10 +73,6 @@ func (e *VersionEndpoint) IsServing() bool {
 	return e.Status == EndpointServing
 }
 
-const (
-	monitoringURLBase = "https://monitoring.dev/graph/d/z0MBKR1Wz/mlp-model-version-dashboard"
-)
-
 type EndpointMonitoringURLParams struct {
 	Cluster      string
 	Project      string

api/service/version_service.go

@@ -24,10 +24,6 @@ import (
 	"github.com/gojek/merlin/models"
 )
 
-const (
-	monitoringBaseURL = "https://monitoring.dev/graph/d/z0MBKR1Wz/mlp-model-version-dashboard"
-)
-
 type VersionsService interface {
 	ListVersions(ctx context.Context, modelId models.Id, monitoringConfig config.MonitoringConfig) ([]*models.Version, error)
 	Save(ctx context.Context, version *models.Version, monitoringConfig config.MonitoringConfig) (*models.Version, error)

charts/merlin/templates/deployment.yaml

@@ -178,6 +178,8 @@ spec:
           {{- if .Values.merlin.monitoring.enabled }}
           - name: MONITORING_DASHBOARD_BASE_URL
             value: "{{ .Values.merlin.monitoring.baseURL }}"
+          - name: MONITORING_DASHBOARD_JOB_BASE_URL
+            value: "{{ .Values.merlin.monitoring.jobBaseURL }}"
           {{- end }}
           - name: ALERT_ENABLED
             value: "{{ .Values.merlin.alert.enabled }}"

charts/merlin/values.yaml

@@ -59,6 +59,8 @@ merlin:
 
   monitoring:
     enabled: false
+    # baseURL: ""
+    # jobBaseURL: ""
 
   warden:
     apiHost: ""

ui/src/config.js

@@ -37,6 +37,9 @@ export const featureToggleConfig = {
     getEnv("REACT_APP_MONITORING_DASHBOARD_ENABLED")
       .toString()
       .toLowerCase() === "false"
+  ),
+  monitoringDashboardJobBaseURL: getEnv(
+    "REACT_APP_MONITORING_DASHBOARD_JOB_BASE_URL"
   )
 };
 

ui/src/job/JobListTable.js

@@ -36,6 +36,7 @@ import StopPredictionJobModal from "./modals/StopPredictionJobModal";
 import { featureToggleConfig } from "../config";
 
 const moment = require("moment");
+const querystring = require("querystring");
 
 const defaultTextSize = "s";
 const defaultIconSize = "xs";
@@ -68,11 +69,19 @@ const JobListTable = ({ projectId, modelId, jobs, isLoaded, error }) => {
     []
   );
 
-  function createMonitoringUrl(project, job) {
+  function createMonitoringUrl(baseURL, project, job) {
     const start_time_nano =
       moment(job.created_at, "YYYY-MM-DDTHH:mm.SSZ").unix() * 1000;
     const end_time_nano = start_time_nano + 7200000;
-    return `https://monitoring.dev/graph/d/_iMaa8RMk/merlin-batch-prediction?orgId=1&from=${start_time_nano}&to=${end_time_nano}&var-cluster=${job.environment.cluster}&var-project=${project.name}&var-job=${job.name}`;
+    const query = {
+      from: start_time_nano,
+      to: end_time_nano,
+      "var-cluster": job.environment.cluster,
+      "var-project": project.name,
+      "var-job": job.name
+    };
+    const queryParams = querystring.stringify(query);
+    return `${baseURL}?${queryParams}`;
   }
 
   const [
@@ -176,7 +185,11 @@ const JobListTable = ({ projectId, modelId, jobs, isLoaded, error }) => {
           {featureToggleConfig.monitoringEnabled && (
             <EuiFlexItem grow={false}>
               <a
-                href={createMonitoringUrl(project.data, item)}
+                href={createMonitoringUrl(
+                  featureToggleConfig.monitoringDashboardJobBaseURL,
+                  project.data,
+                  item
+                )}
                 target="_blank"
                 rel="noopener noreferrer">
                 <EuiButtonEmpty iconType="visLine" size={defaultIconSize}>