Pipelines update storage via an spk deployment command (#95)

* Wrote updatetable logic and created update command

* Adding unit test

* Adding jsdoc

* Adding some docs

* Indenting steps

* Prettier is messing up my formatting of docs

* Address some feedback

* Massive refactor, improvements and bug fixes

* Renamed command based on suggestions

* Updating docs to reflect the changes

* Struggling to improve code coverage

Author: samiyaakhtar

docs/service-introspection-onboarding.md

@@ -92,6 +92,26 @@ To configure the variable group run:
 spk variable-group create
 ```
 
+You will need the following variables:
+
+- `ACCOUNT_KEY`: Set this to the access key for your storage account
+- `ACCOUNT_NAME`: Set this to the name of your storage account
+- `PARTITION_KEY`: This field can be a distinguishing key that recognizea your
+  source repository in the storage, for eg. in this example, we're using the
+  name of the source repository `hello-bedrock`
+- `TABLE_NAME`: Set this to the name of the table in your storage account that
+  you prefer to use
+
+![](./images/variable_group.png)
+
+Make sure that you update the pipelines in the following steps to include this
+variable group, such as below:
+
+```yaml
+variables:
+  - group: <your-variable-group-name>
+```
+
 #### 2. CI pipeline configuration
 
 The CI pipeline runs from the source repository to build a docker image.
@@ -100,15 +120,15 @@ Paste the following task in its corresponding `azure-pipelines.yml`:
 
 ```yaml
 - bash: |
-    curl $SCRIPT > script.sh
-    chmod +x ./script.sh
     tag_name="hello-spektate-$(Build.SourceBranchName)-$(Build.BuildId)"
     commitId=$(Build.SourceVersion)
     commitId=$(echo "${commitId:0:7}")
-    ./script.sh $(ACCOUNT_NAME) $(ACCOUNT_KEY) $(TABLE_NAME) $(PARTITION_KEY) p1 $(Build.BuildId) imageTag $tag_name commitId $commitId service $(Build.Repository.Name)
-  displayName: Update manifest pipeline details in CJ db
-  env:
-    SCRIPT: https://raw.githubusercontent.com/catalystcode/spk/master/scripts/update_introspection.sh
+    VERSION_TO_DOWNLOAD=$(curl -s "https://api.github.com/repos/CatalystCode/spk/releases/latest" | grep "tag_name" | sed -E 's/.*"([^"]+)".*/\1/')
+    echo "Downloading SPK version $VERSION_TO_DOWNLOAD" && wget "https://github.com/CatalystCode/spk/releases/download/$VERSION_TO_DOWNLOAD/spk-linux"
+    chmod +x ./spk-linux
+    ./spk-linux deployment create -n $(ACCOUNT_NAME) -k $(ACCOUNT_KEY) -t $(TABLE_NAME) -p $(PARTITION_KEY) --p1 $(Build.BuildId) --image-tag $tag_name --commit-id $commitId --service $service
+
+  displayName: Update manifest pipeline details in Spektate db
 ```
 
 This task will update the service introspection storage table for every build
@@ -127,58 +147,47 @@ by service introspection.
 
 The CD release pipeline updates the docker image number in the HLD.
 
-Paste the following task in its corresponding `azure-pipelines.yml`:
+Paste the following task towards the end of your release step in the release
+pipeline:
 
 ```yaml
- latest_commit=$(git rev-parse --short HEAD)
- echo "latest_commit=$latest_commit"
-
- # Download update storage script
- curl https://raw.githubusercontent.com/catalystcode/spk/master/scripts/update_introspection.sh > script.sh
- chmod +x script.sh
-
- ./script.sh $(ACCOUNT_NAME) $(ACCOUNT_KEY) $(TABLE_NAME) $(PARTITION_KEY) imageTag $(Build.BuildId) p2 $(Release.ReleaseId) hldCommitId $latest_commit env $(Release.EnvironmentName)
+latest_commit=$(git rev-parse --short HEAD) echo "latest_commit=$latest_commit"
+
+VERSION_TO_DOWNLOAD=$(curl -s
+"https://api.github.com/repos/CatalystCode/spk/releases/latest" | grep
+"tag_name" | sed -E 's/.*"([^"]+)".*/\1/') echo "Downloading SPK version
+$VERSION_TO_DOWNLOAD" && wget
+"https://github.com/CatalystCode/spk/releases/download/$VERSION_TO_DOWNLOAD/spk-linux"
+chmod +x ./spk-linux ./spk-linux deployment create  -n $(ACCOUNT_NAME) -k
+$(ACCOUNT_KEY) -t $(TABLE_NAME) -p $(PARTITION_KEY)  --p2 $(Release.ReleaseId)
+--hld-commit-id $latest_commit --env $(Release.EnvironmentName) --image-tag
+$(Build.BuildId)
 ```
 
-This task is the same as the one from step 1 but instead passes the information
+This task is similar to the one from step 1 but instead passes the information
 that corresponds to the CD release pipeline.
 
 #### 4. HLD manifest pipeline configuration
 
 The HLD manifest pipeline builds the HLD using `fabrikate` and generates
 resource manifests that are then placed in the resource manifest repository.
 
-Paste the following task in the `azure-pipelines.yml` file **before** the
+Paste the following task in the `azure-pipelines.yml` file **after** the
 `fabrikate` steps:
 
 ```yaml
 - bash: |
-    curl $SCRIPT > script.sh
-    chmod +x ./script.sh
+    cd "$HOME"/<name of your manifest repository>
     commitId=$(Build.SourceVersion)
     commitId=$(echo "${commitId:0:7}")
-    ./script.sh $(ACCOUNT_NAME) $(ACCOUNT_KEY) $(TABLE_NAME) $(PARTITION_KEY) hldCommitId $commitId p3 $(Build.BuildId)
-  displayName: Update manifest pipeline details in CJ db
-  env:
-    SCRIPT: https://raw.githubusercontent.com/catalystcode/spk/master/scripts/update_introspection.sh
-```
-
-Paste the following task after the `fabrikate` step:
-
-```yaml
-- script: |
-    cd "$HOME"/hello-bedrock-manifest
-    curl $SCRIPT > script.sh
-    chmod +x ./script.sh
     latest_commit=$(git rev-parse --short HEAD)
-    ./script.sh $(ACCOUNT_NAME) $(ACCOUNT_KEY) $(TABLE_NAME) $(PARTITION_KEY) p3 $(Build.BuildId) manifestCommitId $latest_commit
-  displayName: Update commit id in database
-  env:
-    SCRIPT: https://raw.githubusercontent.com/catalystcode/spk/master/scripts/update_introspection.sh
+    VERSION_TO_DOWNLOAD=$(curl -s "https://api.github.com/repos/CatalystCode/spk/releases/latest" | grep "tag_name" | sed -E 's/.*"([^"]+)".*/\1/')
+    echo "Downloading SPK version $VERSION_TO_DOWNLOAD" && wget "https://github.com/CatalystCode/spk/releases/download/$VERSION_TO_DOWNLOAD/spk-linux"
+    chmod +x ./spk-linux
+    ./spk-linux deployment create -n $(ACCOUNT_NAME) -k $(ACCOUNT_KEY) -t $(TABLE_NAME) -p $(PARTITION_KEY) --p3 $(Build.BuildId) --hld-commit-id $commitId --manifest-commit-id $latest_commit
+  displayName: Update manifest pipeline details in Spektate db
 ```
 
-This task will update the `manifestCommitId`.
-
 ## Getting started
 
 After completing the steps in this guide, you should be able to:

src/commands/deployment/create.ts

@@ -0,0 +1,135 @@
+import commander from "commander";
+import {
+  addSrcToACRPipeline,
+  IDeploymentTable,
+  updateACRToHLDPipeline,
+  updateHLDToManifestPipeline,
+  updateManifestCommitId
+} from "../../lib/azure/deploymenttable";
+import { logger } from "../../logger";
+
+/**
+ * Creates a create command decorator for the command to update a deployment in storage
+ * @param command
+ */
+export const createCommandDecorator = (command: commander.Command): void => {
+  command
+    .command("create")
+    .alias("c")
+    .description("Insert the deployment in storage from pipelines")
+    .option(
+      "-k, --access-key <access-key>",
+      "Access key of the storage account"
+    )
+    .option("-n, --name <account-name>", "Name of the storage account")
+    .option(
+      "-p, --partition-key <partition-key>",
+      "Partition key for the storage account"
+    )
+    .option("-t, --table-name <table-name>", "Name of table in storage account")
+    .option("--p1 <p1>", "Identifier for the first pipeline")
+    .option("--image-tag <image-tag>", "Image tag")
+    .option("--commit-id <commit-id>", "Commit Id in source repository")
+    .option("--service <service>", "Service name")
+    .option("--p2 <p2>", "Identifier for the second pipeline")
+    .option("--hld-commit-id <hld-commit-id>", "Commit id in HLD repository")
+    .option("--env <env>", "Release environment name")
+    .option("--p3 <p3>", "Identifier for the third pipeline")
+    .option(
+      "--manifest-commit-id <manifest-commit-id>",
+      "Commit Id in the manifest repository"
+    )
+    .action(async opts => {
+      if (
+        !opts.accessKey ||
+        !opts.name ||
+        !opts.partitionKey ||
+        !opts.tableName
+      ) {
+        logger.error(
+          "You must specify all of access key, storage account name, partition key, table name, filter name and filter value"
+        );
+        return;
+      }
+
+      const tableInfo: IDeploymentTable = {
+        accountKey: opts.accessKey,
+        accountName: opts.name,
+        partitionKey: opts.partitionKey,
+        tableName: opts.tableName
+      };
+
+      // This is being called from the first pipeline. Make sure all other fields are defined.
+      if (opts.p1) {
+        if (!opts.imageTag || !opts.commitId || !opts.service) {
+          logger.error(
+            "For updating the details of source pipeline, you must specify --image-tag, --commit-id and --service"
+          );
+          return;
+        }
+
+        addSrcToACRPipeline(
+          tableInfo,
+          opts.p1,
+          opts.imageTag,
+          opts.service,
+          opts.commitId
+        );
+        return;
+      }
+
+      // This is being called from the second pipeline. Make sure all other fields are defined.
+      if (opts.imageTag || opts.p2) {
+        if (!opts.p2 || !opts.hldCommitId || !opts.env || !opts.imageTag) {
+          logger.error(
+            "For updating the details of image tag release pipeline, you must specify --p2, --hld-commit-id, --image-tag and --env"
+          );
+          return;
+        }
+        updateACRToHLDPipeline(
+          tableInfo,
+          opts.p2,
+          opts.imageTag,
+          opts.hldCommitId,
+          opts.env
+        );
+        return;
+      }
+
+      // This is being called from the third pipeline. Make sure all other fields are defined.
+      if (opts.hldCommitId && opts.p3) {
+        if (!opts.p3) {
+          logger.error(
+            "For updating the details of manifest generation pipeline, you must specify --p3"
+          );
+          return;
+        }
+        if (opts.manifestCommitId) {
+          updateHLDToManifestPipeline(
+            tableInfo,
+            opts.hldCommitId,
+            opts.p3,
+            opts.manifestCommitId
+          );
+        } else {
+          updateHLDToManifestPipeline(tableInfo, opts.hldCommitId, opts.p3);
+        }
+        return;
+      }
+
+      // This is being called from the third pipeline to update manifest id. Make sure all other fields are defined.
+      if (opts.p3 && opts.manifestCommitId) {
+        if (!opts.manifestCommitId) {
+          logger.error(
+            "For updating the details of manifest generation pipeline, you must specify --manifest-commit-id"
+          );
+          return;
+        }
+        updateManifestCommitId(tableInfo, opts.p3, opts.manifestCommitId);
+        return;
+      }
+
+      //  Execution should not get here; request user to specify arguments correctly
+      logger.error("No action could be performed for specified arguments.");
+    });
+};

src/commands/deployment/index.ts

@@ -1,4 +1,5 @@
 import { Command } from "../command";
+import { createCommandDecorator } from "./create";
 import { dashboardCommandDecorator } from "./dashboard";
 import { getCommandDecorator } from "./get";
 import { onboardCommandDecorator } from "./onboard";
@@ -14,6 +15,7 @@ export const deploymentCommand = Command(
     getCommandDecorator,
     onboardCommandDecorator,
     validateCommandDecorator,
-    dashboardCommandDecorator
+    dashboardCommandDecorator,
+    createCommandDecorator
   ]
 );