diff --git a/bigtable_automation/gcf/custom_dc/README.md b/bigtable_automation/gcf/custom_dc/README.md new file mode 100644 index 00000000..1a9343ba --- /dev/null +++ b/bigtable_automation/gcf/custom_dc/README.md @@ -0,0 +1,18 @@ +# Custom DC BT automation + +## Backgroud + +Custom DC BT automation also triggers KG builders owned by the DC team by writing to a blob in the resource bucket. + +The blob written to the following path will trigger a cache build. +```sh +gs:///..//process//trigger.txt +``` + +## How to test + +From custom_dc directory, run `./deploy_local.sh` to start custom DC gcf locally. + +### Testing csv trigger + +From gcf directory, run `./custom_dc/test.sh publish`. diff --git a/bigtable_automation/gcf/custom_dc/deploy_local.sh b/bigtable_automation/gcf/custom_dc/deploy_local.sh new file mode 100755 index 00000000..630804e3 --- /dev/null +++ b/bigtable_automation/gcf/custom_dc/deploy_local.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR + +# Pick environment config file +config_file=local.yaml + +# Good to read the yaml file keys and convert them to bash array +for var in projectID cluster instance dataflowTemplate tempLocation controllerTriggerTopic bucket +do + value=$(yq eval .$var $config_file) + export $var=$value +done + +go run main.go \ No newline at end of file diff --git a/bigtable_automation/gcf/custom_dc/import_notification_pubsub_msg_want.txt b/bigtable_automation/gcf/custom_dc/import_notification_pubsub_msg_want.txt new file mode 100644 index 00000000..86e66656 --- /dev/null +++ b/bigtable_automation/gcf/custom_dc/import_notification_pubsub_msg_want.txt @@ -0,0 +1,6 @@ +import_name=IEA_PRIVATE,\ +dc_manifest_path=/memfile/core_resolved_mcfs_memfile/core_resolved_mcfs.binarypb,\ +custom_manifest_path=/bigstore/automation_control_test/user/IEA_PRIVATE/config/config.textproto,\ +bigstore_data_directory=/bigstore/automation_control_test/user/IEA_PRIVATE/tmcf_csv,\ +bigstore_cache_directory=/bigstore/automation_control_test/user/IEA_PRIVATE/cache,\ +bigstore_control_directory=/bigstore/automation_control_test/user/IEA_PRIVATE/control,\ diff --git a/bigtable_automation/gcf/custom_dc/local.yaml b/bigtable_automation/gcf/custom_dc/local.yaml new file mode 100644 index 00000000..849d1e6c --- /dev/null +++ b/bigtable_automation/gcf/custom_dc/local.yaml @@ -0,0 +1,7 @@ +projectID: "google.com:datcom-store-dev" +instance: "prophet-test" +cluster: "prophet-test-c1" +dataflowTemplate: "gs://datcom-dataflow-templates/templates/flex/csv_to_bt.json" +tempLocation: "gs://datcom-store-dev-resources/dataflow/tmp" +controllerTriggerTopic: "projects/datcom-204919/topics/custom-import-notification-test" +bucket: "automation_control_test" diff --git a/bigtable_automation/gcf/custom_dc/main.go b/bigtable_automation/gcf/custom_dc/main.go new file mode 100644 index 00000000..15046c82 --- /dev/null +++ b/bigtable_automation/gcf/custom_dc/main.go @@ -0,0 +1,43 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log" + "os" + + "github.com/GoogleCloudPlatform/functions-framework-go/funcframework" + "github.com/datacommonsorg/tools/bigtable_automation/gcf" +) + +func main() { + ctx := context.Background() + if err := funcframework.RegisterEventFunctionContext( + ctx, + "/", + gcf.CustomBTImportController, + ); err != nil { + log.Fatalf("funcframework.RegisterEventFunctionContext: %v\n", err) + } + // Use PORT environment variable, or default to 8080. + port := "8080" + if envPort := os.Getenv("PORT"); envPort != "" { + port = envPort + } + if err := funcframework.Start(port); err != nil { + log.Fatalf("funcframework.Start: %v\n", err) + } +} diff --git a/bigtable_automation/gcf/custom_dc/test.sh b/bigtable_automation/gcf/custom_dc/test.sh new file mode 100755 index 00000000..34d8cd11 --- /dev/null +++ b/bigtable_automation/gcf/custom_dc/test.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test GCF locally. +set -x + +if [[ $# != 1 ]]; then + echo "Usage: $0 (publish)" >&2 + exit 1 +fi + +# Test import notification pubsub message has the correct format. +# Mock a GCS file event to local GCF while publishing to a real PubSub topic. +if [[ "$1" == "publish" ]]; then + curl localhost:8080 \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "context": { + "eventId": "1147091835525187", + "timestamp": "2021-01-23T07:38:57.772Z", + "eventType": "google.storage.object.finalize", + "resource": { + "service": "storage.googleapis.com", + "name": "projects/_/buckets/automation_control_test/user/IEA_PRIVATE/process/import1/trigger.txt", + "type": "storage#object" + } + }, + "data": { + "bucket": "automation_control_test", + "contentType": "text/plain", + "kind": "storage#object", + "md5Hash": "...", + "metageneration": "1", + "name": "user/IEA_PRIVATE/process/import1/trigger.txt", + "size": "0", + "storageClass": "MULTI_REGIONAL", + "timeCreated": "2020-04-23T07:38:57.230Z", + "timeStorageClassUpdated": "2020-04-23T07:38:57.230Z", + "updated": "2020-04-23T07:38:57.230Z" + } + }' && sleep 5 + + # Note: This should match the topic in local.yaml + SUB=projects/datcom-204919/subscriptions/custom-import-notification-test-sub + + GOT=$(gcloud pubsub subscriptions pull $SUB --auto-ack --format=json | jq -r .[].message.data | base64 -d) + WANT=$(cat custom_dc/import_notification_pubsub_msg_want.txt) + + set +x + if [[ "$WANT" == "$GOT" ]]; then + echo "###########################################################################" + echo "Test passed: Publish to controller got expected pubusb message." + echo "###########################################################################" + else + echo "###########################################################################" + echo "Test failed: Publish to controller, got unexpected pubsub message from gcf." + echo "###########################################################################" + fi + +fi