Push to the live registry after publishing

Author: iwahbe

.github/workflows/pull-request.yml

@@ -19,6 +19,7 @@ jobs:
     - lint-markdown
     - lint-scripts
     - preview
+    - test-live-publish
     runs-on: ubuntu-latest
     steps:
     - uses: guibranco/github-status-action-v2@0849440ec82c5fa69b2377725b9b7852a3977e76 # v1.1.13
@@ -77,6 +78,18 @@ jobs:
       - name: Run Linter
         run: yarn run lint
 
+  test-live-publish:
+    name: Test Live Registry Publish
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out branch
+        uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - run: uv run --with requests,pyyaml scripts/ci/push-registry.py --dry-run
+        env:
+          PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+
   # Preview runs a registry build into a commit specific S3 bucket to preview changes.
   #
   # A link to the generated build is appended to the PR on each commit.

.github/workflows/push.yml

@@ -64,6 +64,12 @@ jobs:
         with:
           name: origin-bucket-metadata
           path: origin-bucket-metadata.json
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Push to the Live Registry
+        run: uv run --with requests,pyyaml ./scripts/ci/push-registry.py
+        env:
+          PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
 
   notify:
     if: failure()

scripts/push-registry.py renamed to scripts/ci/push-registry.py

@@ -2,122 +2,203 @@
 
 # Can be run without dependencies with
 #
-#	uv run --with requests,pyyaml scripts/push-registry.py
+#	uv run --with requests,pyyaml scripts/ci/push-registry.py
 
 import os
 import tempfile
 import requests
+import subprocess
 import json
 import yaml
-import sys
-from logging import info, basicConfig, INFO
+import argparse
+from logging import info, warning, basicConfig, WARNING
+
+basicConfig(level=WARNING)
 
 yaml_db_path = "themes/default/data/registry/packages"
 docs_path = "themes/default/content/registry/packages"
+backend_url = os.getenv("PULUMI_BACKEND_URL") or "https://api.pulumi.com/api"
 tmpdir = tempfile.mkdtemp()
 
-basicConfig(level=INFO)
+token = os.getenv("PULUMI_ACCESS_TOKEN")
 
-info(f"Tempdir: {tmpdir}")
+parser = argparse.ArgumentParser(
+    prog='push-registry',
+    description="""Push the contents of the pulumi/registry GitHub repo to the live registry service
+
+    PULUMI_ACCESS_TOKEN must be set to publish.
+    PULUMI_BACKEND_URL may be set. Defaults to "https://api.pulumi.com/api".
+    """,
+    epilog='This is a Pulumi internal tool - it is not intended for external use')
+parser.add_argument("--target", default=None, help='Only focus on one provider. This should just be the name, like "aws"')
+parser.add_argument("--dry-run", action='store_true', help="Don't actually publish provider, but do everything else")
 
-run_script = []
+args = parser.parse_args()
 
-target = None
-if len(sys.argv) >= 2:
-    target = sys.argv[1]
+info(f"Tempdir: {tmpdir}")
 
 publishers = {
     "1Password": "1Password",
-    "airbytehq": "airbytehq",
     "Aviatrix": "Aviatrix",
-    "azaurus1": "azaurus1",
-    "castai": "castai",
-    "celest-dev": "celest-dev",
+    "CTFer.io": "ctfer",
     "Checkly": "checkly",
     "Christian Nunciato": "christian_nunciato",
     "Chronosphere": "chronosphere",
     "Cisco": "cisco",
-    "civo": "civo",
-    "constellix": "constellix",
     "CrowdStrike": "crowdstrike",
-    "CTFer.io": "ctfer",
     "Daniel Muehlbachler-Pietrzykowski": "daniel_muehlbachler_pietrzykowski",
     "DataRobot, Inc.": "datarobot",
     "Defang": "defang",
     "Descope": "descope",
-    "dirien": "dirien",
-    "dmacvicar": "dmacvicar",
     "Equinix": "equinix",
     "EventStore": "eventstore",
+    "Genesiscloud": "genesiscloud",
+    "Grapl Security": "grapl_security",
+    "Ian Wahbe": "ian_wahbe",
+    "Impart Security": "impart_security",
+    "Koyeb": "koyeb",
+    "Lee Zen": "lee_zen",
+    "Nuage-Studio": "nuage_studio",
+    "OVHcloud": "ovhcloud",
+    "Paul Stack": "paul_stack",
+    "Piers Karsenbarg": "piers_karsenbarg",
+    "Prodvana": "prodvana",
+    "Pulumi": "pulumi",
+    "Pulumiverse": "pulumiverse",
+    "RedisLabs": "redislabs",
+    "Rootly": "rootly",
+    "Runpod": "runpod",
+    "Symbiosis": "symbiosis",
+    "Theo Gravity": "theo_gravity",
+    "Three141": "three141",
+    "Threefold": "threefold",
+    "Twingate": "twingate",
+    "UpCloudLtd": "upcloudltd",
+    "Upstash": "upstash",
+    "Vates": "vates",
+    "Volcengine": "volcengine",
+    "Wttech": "wttech",
+    "Zscaler": "zscaler",
+    "airbytehq": "airbytehq",
+    "akeyless-community": "akeyless-community",
+    "aptible": "aptible",
+    "athenz": "athenz",
+    "azaurus1": "azaurus1",
+    "castai": "castai",
+    "celest-dev": "celest-dev",
+    "chainguard-dev": "chainguard-dev",
+    "checkpointsw": "checkpointsw",
+    "ciscodevnet": "ciscodevnet",
+    "ciscodevnet": "ciscodevnet",
+    "civo": "civo",
+    "cloudfoundry-community": "cloudfoundry-community",
+    "coder": "coder",
+    "constellix": "constellix",
+    "coralogix": "coralogix",
+    "cox-automotive": "cox-automotive",
+    "cyralinc": "cyralinc",
+    "datadrivers": "datadrivers",
+    "dell": "dell",
+    "dell": "dell",
+    "dell": "dell",
+    "denouche": "denouche",
+    "dirien": "dirien",
+    "dmacvicar": "dmacvicar",
+    "dome9": "dome9",
+    "drfaust92": "drfaust92",
+    "e-breuninger": "e-breuninger",
+    "edge-center": "edge-center",
+    "elastic": "elastic",
+    "elastic-infra": "elastic-infra",
+    "ferlab-ste-justine": "ferlab-ste-justine",
+    "ferlab-ste-justine": "ferlab-ste-justine",
     "fivetran": "fivetran",
+    "flexibleenginecloud": "flexibleenginecloud",
     "fortinetdev": "fortinetdev",
-    "Genesiscloud": "genesiscloud",
+    "g-core": "g-core",
+    "glesys": "glesys",
     "goauthentik": "goauthentik",
-    "Grapl Security": "grapl_security",
     "hashicorp": "hashicorp",
     "honeycombio": "honeycomb",
-    "Ian Wahbe": "ian_wahbe",
-    "Impart Security": "impart_security",
+    "hpe": "hpe",
+    "ibm-cloud": "ibm-cloud",
+    "imperva": "imperva",
     "incident-io": "incident_io",
+    "infobloxopen": "infobloxopen",
+    "ionos-cloud": "ionos-cloud",
+    "iterative": "iterative",
+    "jdamata": "jdamata",
     "jfrog": "jfrog",
     "joneshf": "joneshf",
+    "juju": "juju",
+    "k-yomo": "k-yomo",
     "komminarlabs": "komminarlabs",
     "kong": "kong",
-    "Koyeb": "koyeb",
     "labd": "labd",
+    "lacework": "lacework",
     "lbrlabs": "lbrlabs",
-    "Lee Zen": "lee_zen",
     "littlejo": "littlejo",
+    "logdna": "logdna",
+    "logzio": "logzio",
     "lucky3028": "lucky3028",
+    "mastercard": "mastercard",
+    "maxlaverse": "maxlaverse",
+    "megaport": "megaport",
+    "mrolla": "mrolla",
+    "nats-io": "nats-io",
+    "netapp": "netapp",
     "netlify": "netlify",
-    "Nuage-Studio": "nuage_studio",
+    "octopusdeploylabs": "octopusdeploylabs",
     "onelogin": "onelogin",
+    "opennebula": "opennebula",
+    "opensearch-project": "opensearch-project",
+    "opentelekomcloud": "opentelekomcloud",
     "oun": "oun",
     "outscale": "outscale",
-    "OVHcloud": "ovhcloud",
-    "Paul Stack": "paul_stack",
+    "paloaltonetworks": "paloaltonetworks",
+    "paloaltonetworks": "paloaltonetworks",
+    "pan-net": "pan-net",
+    "paultyng": "paultyng",
     "pgEdge": "pgedge",
-    "Piers Karsenbarg": "piers_karsenbarg",
+    "philips-software": "philips-software",
     "pinecone-io": "pinecone",
     "planetscale": "planetscale",
     "port-labs": "port_labs",
     "prefecthq": "prefecthq",
-    "Prodvana": "prodvana",
     "propelauth": "propelauth",
-    "Pulumi": "pulumi",
     "pulumiverse - Marcel Arns": "pulumiverse",
     "pulumiverse": "pulumiverse",
-    "Pulumiverse": "pulumiverse",
     "qdrant": "qdrant",
     "rancher": "rancher",
-    "RedisLabs": "redislabs",
     "redpanda-data": "redpanda_data",
-    "Rootly": "rootly",
-    "Runpod": "runpod",
+    "rollbar": "rollbar",
+    "selectel": "selectel",
+    "spectrocloud": "spectrocloud",
     "splightplatform": "splightplatform",
     "supabase": "supabase",
-    "Symbiosis": "symbiosis",
+    "sysdiglabs": "sysdiglabs",
     "temporalio": "temporalio",
+    "tencentcloudstack": "tencentcloudstack",
     "terraform-lxd": "terraform_lxd",
-    "Theo Gravity": "theo_gravity",
-    "Three141": "three141",
-    "Threefold": "threefold",
+    "terraform-routeros": "terraform-routeros",
     "timescale": "timescale",
-    "Twingate": "twingate",
-    "UpCloudLtd": "upcloudltd",
-    "Upstash": "upstash",
+    "ucloud": "ucloud",
     "vantage-sh": "vantage-sh",
-    "Vates": "vates",
-    "Volcengine": "volcengine",
-    "Wttech": "wttech",
+    "vk-cs": "vk-cs",
+    "vmware": "vmware",
+    "vmware": "vmware",
+    "vmware": "vmware",
+    "vmware": "vmware",
+    "vmware": "vmware",
     "zenduty": "zenduty",
-    "Zscaler": "zscaler",
 }
 
+has_failed = False
+
 for filename in os.listdir(yaml_db_path):
     if not filename.endswith('.yaml'):
         continue
-    if target and target != filename.removesuffix(".yaml"):
+    if args.target and args.target != filename.removesuffix(".yaml"):
         continue
 
     filepath = os.path.join(yaml_db_path, filename)
@@ -140,11 +221,11 @@
     if publisher_display == "DEPRECATED":
         continue
     elif publisher_display not in publishers:
-        raise Exception(f"Missing publisher entry for {publisher_display}")
+        raise Exception(f'Missing publisher entry for "{publisher_display}"')
     publisher = publishers[publisher_display]
 
 
-    package_version_name = f"{source}/{publisher}/{name}@{version}"
+    package_version_name = f"{source}/{publisher}/{name}@{version.removeprefix("v")}"
     info(f"Preparing command for {package_version_name}")
 
     # We skip azure-native-v1, since it will (1) never be updated again and (2) isn't
@@ -155,8 +236,11 @@
     if name.startswith("azure-native") and name != "azure-native":
         continue
 
-    api_url = f"https://api.pulumi.com/api/registry/preview/packages/{source}/{publisher}/{name}/versions/{version}"
-    existence_check = requests.head(api_url).status_code
+
+    api_url = f"{backend_url}/preview/registry/packages/{source}/{publisher}/{name}/versions/{version.removeprefix("v")}"
+    existence_check = requests.get(api_url, headers={
+        "Authorization": f"token {token}",
+    }).status_code
 
     if existence_check == 404:
         schema_file = os.path.join(tmpdir, name, "schema.json")
@@ -173,26 +257,41 @@
             else:
                 schema = json.loads(response.content)
 
-            if "version" not in schema:
+
+            # We *correct* the version if it isn't present *or* if it disagrees with the
+            # version that has been published in github.com/pulumi/registry.
+            if "version" not in schema or schema["version"] != version.removeprefix("v"):
                 schema["version"] = version.removeprefix("v")
             json.dump(schema, sf)
 
+
         cmd = [
-            "pulumi", "package", "publish", schema_file,
-            f"--readme={docs_path}/{name}/_index.md",
-            f"--source={source}",
-            f"--publisher={publisher}",
+            "pulumi", "package", "publish",
+            schema_file,
+            "--readme", f"{docs_path}/{name}/_index.md",
+            "--source", source,
+            "--publisher", publisher,
         ]
 
         if os.path.isfile(os.path.join(docs_path, name, "_installation-configuration.md")):
             cmd.extend(["--installation-configuration", f"{docs_path}/{name}/installation-configuration.md"])
 
-        run_script.append(' '.join(cmd))
+        if args.dry_run:
+            print(" ".join(cmd))
+        else:
+            try:
+                subprocess.run(cmd, shell=False, check=True)
+            except subprocess.CalledProcessError:
+                warning(f"Failed to run {" ".join(cmd)}")
+                has_failed=True
+
+        os.remove(schema_file)
+        os.removedirs(os.path.dirname(schema_file))
     elif existence_check == 200:
         info("Package already exists in the registry DB")
     else:
-        info(f"Unable to check on package {package_version_name}")
-        exit(1)
+        warning(f"Unable to check on package {package_version_name}")
+        has_failed=True
 
-for cmd in run_script:
-    print(cmd)
+if has_failed:
+    exit(1)