[Custom DC] Add local gcf for csv trigger and basic tests (#182)

* [Custom DC] Add local gcf for csv trigger and basic tests

* Update tests

* Update doc

---------

Co-authored-by: Alex Chen <[email protected]>

bigtable_automation/gcf/custom_dc/README.md

@@ -0,0 +1,18 @@
+# Custom DC BT automation
+
+## Backgroud
+
+Custom DC BT automation also triggers KG builders owned by the DC team by writing to a blob in the resource bucket.
+
+The blob written to the following path will trigger a cache build.
+```sh
+gs://<resource-bucket>/../<import name>/process/<import id>/trigger.txt
+```
+
+## How to test
+
+From custom_dc directory, run `./deploy_local.sh` to start custom DC gcf locally.
+
+### Testing csv trigger
+
+From gcf directory, run `./custom_dc/test.sh publish`.

bigtable_automation/gcf/custom_dc/deploy_local.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd $DIR
+
+# Pick environment config file
+config_file=local.yaml
+
+# Good to read the yaml file keys and convert them to bash array
+for var in projectID cluster instance dataflowTemplate tempLocation controllerTriggerTopic bucket
+do
+    value=$(yq eval .$var $config_file)
+    export $var=$value
+done
+
+go run main.go

bigtable_automation/gcf/custom_dc/import_notification_pubsub_msg_want.txt

@@ -0,0 +1,6 @@
+import_name=IEA_PRIVATE,\
+dc_manifest_path=/memfile/core_resolved_mcfs_memfile/core_resolved_mcfs.binarypb,\
+custom_manifest_path=/bigstore/automation_control_test/user/IEA_PRIVATE/config/config.textproto,\
+bigstore_data_directory=/bigstore/automation_control_test/user/IEA_PRIVATE/tmcf_csv,\
+bigstore_cache_directory=/bigstore/automation_control_test/user/IEA_PRIVATE/cache,\
+bigstore_control_directory=/bigstore/automation_control_test/user/IEA_PRIVATE/control,\

bigtable_automation/gcf/custom_dc/local.yaml

@@ -0,0 +1,7 @@
+projectID: "google.com:datcom-store-dev"
+instance: "prophet-test"
+cluster: "prophet-test-c1"
+dataflowTemplate: "gs://datcom-dataflow-templates/templates/flex/csv_to_bt.json"
+tempLocation: "gs://datcom-store-dev-resources/dataflow/tmp"
+controllerTriggerTopic: "projects/datcom-204919/topics/custom-import-notification-test"
+bucket: "automation_control_test"

bigtable_automation/gcf/custom_dc/main.go

@@ -0,0 +1,43 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+
+	"github.com/GoogleCloudPlatform/functions-framework-go/funcframework"
+	"github.com/datacommonsorg/tools/bigtable_automation/gcf"
+)
+
+func main() {
+	ctx := context.Background()
+	if err := funcframework.RegisterEventFunctionContext(
+		ctx,
+		"/",
+		gcf.CustomBTImportController,
+	); err != nil {
+		log.Fatalf("funcframework.RegisterEventFunctionContext: %v\n", err)
+	}
+	// Use PORT environment variable, or default to 8080.
+	port := "8080"
+	if envPort := os.Getenv("PORT"); envPort != "" {
+		port = envPort
+	}
+	if err := funcframework.Start(port); err != nil {
+		log.Fatalf("funcframework.Start: %v\n", err)
+	}
+}

bigtable_automation/gcf/custom_dc/test.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test GCF locally.
+set -x
+
+if [[ $# != 1 ]]; then
+  echo "Usage: $0 (publish)" >&2
+  exit 1
+fi
+
+# Test import notification pubsub message has the correct format.
+# Mock a GCS file event to local GCF while publishing to a real PubSub topic.
+if [[ "$1" == "publish" ]]; then
+  curl localhost:8080 \
+    -X POST \
+    -H "Content-Type: application/json" \
+    -d '{
+        "context": {
+          "eventId": "1147091835525187",
+          "timestamp": "2021-01-23T07:38:57.772Z",
+          "eventType": "google.storage.object.finalize",
+          "resource": {
+             "service": "storage.googleapis.com",
+             "name": "projects/_/buckets/automation_control_test/user/IEA_PRIVATE/process/import1/trigger.txt",
+             "type": "storage#object"
+          }
+        },
+        "data": {
+          "bucket": "automation_control_test",
+          "contentType": "text/plain",
+          "kind": "storage#object",
+          "md5Hash": "...",
+          "metageneration": "1",
+          "name": "user/IEA_PRIVATE/process/import1/trigger.txt",
+          "size": "0",
+          "storageClass": "MULTI_REGIONAL",
+          "timeCreated": "2020-04-23T07:38:57.230Z",
+          "timeStorageClassUpdated": "2020-04-23T07:38:57.230Z",
+          "updated": "2020-04-23T07:38:57.230Z"
+        }
+      }' && sleep 5
+
+  # Note: This should match the topic in local.yaml
+  SUB=projects/datcom-204919/subscriptions/custom-import-notification-test-sub
+
+  GOT=$(gcloud pubsub subscriptions pull $SUB --auto-ack --format=json | jq  -r .[].message.data | base64 -d)
+  WANT=$(cat custom_dc/import_notification_pubsub_msg_want.txt)
+
+  set +x
+  if [[ "$WANT" == "$GOT" ]]; then
+    echo "###########################################################################"
+    echo "Test passed: Publish to controller got expected pubusb message."
+    echo "###########################################################################"
+  else
+    echo "###########################################################################"
+    echo "Test failed: Publish to controller, got unexpected pubsub message from gcf."
+    echo "###########################################################################"
+  fi
+
+fi