chore: replace yarn with npm (#5128)

* chore: replace yarn with npm

* fix: parameter passing to `npm run`

* fix: cache node_modules folder

* fix: release build script for npm

* chore: cache all node_modules folders

* fix: pass e2e options along

* fix: broken workspaces config

* fix: npm install

* fix: nohoist in package-lock

* fix: correct package.lock nesting

* fix: cache node_modules for plugins

* fix: split2 dependency

* fix: more missing dependencies

* fix: dependencies dependencies dependencies

* fix: more dependencies

* fix: and some more dependencies

* fix: you guessed it...

* fix: yup

* chore: cleanup plugins and sdk build outputs

* fix: even more missing dependencies

* fix: more dependency and module stuff

* fix: move sdk outputs into build directory and do not hide fact that this is not an ESM module so this only worked because we bundled everything into the same directory

* refactor: you will not believe it

* refactor: how many dependencies were never really declared?

* refactor: touch was listed twice

* fix: CJS compatible node-fetch

* fix: make mocha find tests

* fix: more missing dependencies

* fix: prettier fix

* chore: add package-lock for k8s-namespace and k8s-deployment package

* fix: by now I am out of ideas

* fix: sdk tsconfig

* fix: pulumi test files are not in build folder

* chore: move pulumi plugin into src folder

* chore: move all plugin code in src directories

* fix: make sure plugin tests can find their static test files

* fix: script-utils ignore

* fix: more plugin test path fixes

* fix: package build using package-lock and workspaces for package install

* fix: alpine builder image

Author: TimBeyer

.circleci/config.yml

@@ -94,14 +94,25 @@ commands:
     steps:
       - restore_cache:
           keys:
-            - yarn-v7-<<parameters.context>>-{{ checksum "yarn.lock" }}
+            - npm-v2-<<parameters.context>>-{{ checksum "package-lock.json" }}
 
-      - run: yarn
+      - run: npm ci
 
       - save_cache:
-          key: yarn-v7-<<parameters.context>>-{{ checksum "yarn.lock" }}
+          key: npm-v2-<<parameters.context>>-{{ checksum "package-lock.json" }}
           paths:
-            - ~/.cache/yarn
+            - node_modules
+            - cli/node_modules
+            - core/node_modules
+            - e2e/node_modules
+            - plugins/conftest/node_modules
+            - plugins/conftest-container/node_modules
+            - plugins/conftest-kubernetes/node_modules
+            - plugins/jib/node_modules
+            - plugins/maven-container/node_modules
+            - plugins/pulumi/node_modules
+            - plugins/terraform/node_modules
+            - sdk/node_modules
 
   configure_git:
     description: Configure git (needed for some tests)
@@ -233,7 +244,7 @@ commands:
       - *attach-workspace
       - restore_cache:
           keys:
-            - pkg-v3-{{ checksum "yarn.lock" }}
+            - pkg-v3-{{ checksum "package-lock.json" }}
       - run:
           name: Install Codesigning utility (needed for ARM64 on MacOS)
           command: |
@@ -248,15 +259,15 @@ commands:
             sudo apt-get install -y binfmt-support
       - run:
           name: Run dist script
-          command: yarn run dist --version "<<parameters.version>>"
+          command: npm run dist -- --version "<<parameters.version>>"
       - persist_to_workspace:
           root: ./
           paths:
             - dist/
             # Needed for the alpine docker build
             - tmp/pkg/
       - save_cache:
-          key: pkg-v3-{{ checksum "yarn.lock" }}
+          key: pkg-v3-{{ checksum "package-lock.json" }}
           paths:
             - tmp/pkg-fetch
       - run:
@@ -279,7 +290,7 @@ jobs:
       - run:
           name: build
           command: |
-            yarn run build
+            npm run build
       # Save the built output to be used for the docker image and for the release steps
       - persist_to_workspace:
           # Must be an absolute path, or relative path from working_directory. This is a directory on the container which is
@@ -316,10 +327,10 @@ jobs:
       - *attach-workspace
       - run:
           name: lint
-          command: yarn run lint
+          command: npm run lint
       - run:
           name: prettier
-          command: yarn run prettier
+          command: npm run prettier
 
   check-docs:
     <<: *node-config
@@ -329,7 +340,7 @@ jobs:
       - *attach-workspace
       - run:
           name: Make sure generated docs are up-to-date
-          command: yarn run check-docs
+          command: npm run check-docs
 
   check-package-licenses:
     <<: *node-config
@@ -341,7 +352,7 @@ jobs:
           name: Make sure dependency licenses are okay
           command: |
             mkdir tmp
-            yarn check-package-licenses
+            npm run check-package-licenses
 
   test-framework:
     <<: *node-config
@@ -355,7 +366,7 @@ jobs:
       - run:
           name: Unit test framework components
           command: |
-            yarn run test:framework:unit
+            npm run test:framework:unit
           environment:
             CHOKIDAR_USEPOLLING: "1"
             NODE_OPTIONS: --max_old_space_size=4096
@@ -382,7 +393,7 @@ jobs:
             cd e2e/projects/<<parameters.project>>
             git checkout .
             cd ~/project
-            yarn e2e-project --project=<<parameters.project>> --env=<<parameters.environment>> --showlog -b
+            npm run e2e-project -- --project=<<parameters.project>> --env=<<parameters.environment>> --showlog -b
       - cleanup_remote_cluster:
           filter: <<parameters.project>>-testing-ci-$CIRCLE_BUILD_NUM-e2e
 
@@ -579,7 +590,7 @@ jobs:
       # Run tests
       - run:
           name: Plugin tests
-          command: yarn run test:plugins
+          command: npm run test:plugins
       # Persist Maven cache
       - save_cache:
           key: m2-v1
@@ -613,7 +624,7 @@ jobs:
             # Notes:
             # - We skip tests that only work for remote environments
             # - We currently don't support in-cluster building on kind.
-            yarn integ-kind
+            npm run integ-kind
       - run:
           name: Deploy demo-project
           command: ./bin/garden deploy --root examples/demo-project --logger-type basic
@@ -742,7 +753,7 @@ jobs:
           # Note: We skip tests that only work for remote environments
           command: |
             cd core
-            GARDEN_SKIP_TESTS="<<parameters.skipTests>>" yarn _integ
+            GARDEN_SKIP_TESTS="<<parameters.skipTests>>" npm run _integ
       - run:
           name: Deploy demo-project
           command: ./bin/garden deploy --root examples/demo-project
@@ -782,7 +793,7 @@ jobs:
           # Note: We skip tests that only work for remote environments
           command: |
             cd core
-            GARDEN_SKIP_TESTS="<<parameters.skipTests>>" yarn _integ
+            GARDEN_SKIP_TESTS="<<parameters.skipTests>>" npm run _integ
       - run:
           name: Deploy demo-project
           command: ./bin/garden deploy --root examples/demo-project

.github/workflows/publish-release.yml

@@ -22,8 +22,8 @@ jobs:
           node-version: 18.x
       - name: Build package
         run: |
-          yarn
-          yarn build
+          npm ci
+          npm run build
       - name: Setup npm registry authentication
         working-directory: ./core
         run: |

.npmrc

@@ -0,0 +1,3 @@
+# Ensure that dependencies are not hoisted up
+# Hoisting may cause troubles with some bundler tools
+install-strategy = "nested"

.yarnrc

@@ -29,9 +29,12 @@
     "@garden-io/garden-jib": "*",
     "@garden-io/garden-pulumi": "*",
     "@garden-io/garden-terraform": "*",
+    "@scg82/exit-hook": "^3.4.1",
     "chalk": "^4.1.2",
+    "node-abi": "^3.45.0",
     "patch-package": "^6.5.1",
-    "pkg": "5.8.1"
+    "pkg": "5.8.1",
+    "source-map-support": "^0.5.21"
   },
   "devDependencies": {
     "@types/chai": "^4.3.4",
@@ -50,8 +53,8 @@
   },
   "scripts": {
     "add-version-files": "node build/src/add-version-files.js",
-    "build": "tsc --build . --verbose && yarn run add-version-files && yarn run generate-docs",
-    "check-package-lock": "git diff-index --quiet HEAD -- yarn.lock || (echo 'yarn.lock is dirty!' && exit 1)",
+    "build": "tsc --build . --verbose && npm run add-version-files && npm run generate-docs",
+    "check-package-lock": "git diff-index --quiet HEAD -- package-lock.json || (echo 'package-lock.json is dirty!' && exit 1)",
     "clean": "shx rm -rf build dist",
     "fix-format": "prettier --write \"{src,test}/**/*.ts\"",
     "generate-docs": "node ./build/src/generate-docs.js",
@@ -66,10 +69,5 @@
       "node_modules/@garden-io/**/*.js",
       "node_modules/es-get-iterator/**/*.js"
     ]
-  },
-  "workspaces": {
-    "nohoist": [
-      "pkg"
-    ]
   }
-}
+}

cli/package.json

@@ -71,6 +71,12 @@ export function getTarballFilename(version: string, targetName: string): string
   return composePackageFilename(version, targetName, "tar.gz")
 }
 
+export type NPMWorkspaceQueryResult = {
+  name: string
+  location: string
+  dependencies: Record<string, string>
+}
+
 async function buildBinaries(args: string[]) {
   const argv = minimist(args)
   const version = argv.version || getPackageVersion()
@@ -90,14 +96,14 @@ async function buildBinaries(args: string[]) {
 
   // Copy each package to the temp dir
   console.log(chalk.cyan("Getting package info"))
-  const res = (await exec("yarn", ["--json", "workspaces", "info"])).stdout
-  const workspaces = JSON.parse(JSON.parse(res).data)
+  const res = (await exec("npm", ["query", ".workspace"])).stdout
+  const workspaces: NPMWorkspaceQueryResult[] = JSON.parse(res)
 
   console.log(chalk.cyan("Copying packages"))
   await Promise.all(
-    Object.entries(workspaces).map(async ([name, info]: [string, any]) => {
-      const sourcePath = resolve(repoRoot, info.location)
-      const targetPath = resolve(tmpDirPath, info.location)
+    workspaces.map(async ({ name, location }: { name: string; location: string }) => {
+      const sourcePath = resolve(repoRoot, location)
+      const targetPath = resolve(tmpDirPath, location)
       await remove(targetPath)
       await mkdirp(targetPath)
       await exec("rsync", [
@@ -117,13 +123,19 @@ async function buildBinaries(args: string[]) {
   // Edit all the packages to have them directly link any internal dependencies
   console.log(chalk.cyan("Modifying package.json files for direct installation"))
   await Promise.all(
-    Object.entries(workspaces).map(async ([name, info]: [string, any]) => {
-      const packageRoot = resolve(tmpDirPath, info.location)
+    workspaces.map(async ({ name, location, dependencies }) => {
+      const packageRoot = resolve(tmpDirPath, location)
       const packageJsonPath = resolve(packageRoot, "package.json")
       const packageJson = require(packageJsonPath)
 
-      for (const depName of info.workspaceDependencies) {
-        const depInfo = workspaces[depName]
+      const workspaceDependencies = Object.keys(dependencies).filter((dependencyName) => {
+        return workspaces.some((p) => p.name === dependencyName)
+      })
+      for (const depName of workspaceDependencies) {
+        const depInfo = workspaces.find((p) => p.name === depName)
+        if (!depInfo) {
+          throw new Error("Could not find workspace info for " + depName)
+        }
         const targetRoot = resolve(tmpDirPath, depInfo.location)
         const relPath = relative(packageRoot, targetRoot)
         packageJson.dependencies[depName] = "file:" + relPath
@@ -141,11 +153,16 @@ async function buildBinaries(args: string[]) {
     })
   )
 
-  // Run yarn install in the cli package
-  console.log(chalk.cyan("Installing packages in @garden-io/cli package"))
-  const cliPath = resolve(tmpDirPath, workspaces["@garden-io/cli"].location)
-  await exec("yarn", ["--production"], { cwd: cliPath })
+  // Run npm install in the cli package
+  await copy(resolve(repoRoot, "package.json"), resolve(tmpDirPath, "package.json"))
+  await copy(resolve(repoRoot, "package-lock.json"), resolve(tmpDirPath, "package-lock.json"))
+  // The `.npmrc` config ensures that we're not hoisting any dependencies
+  await copy(resolve(repoRoot, ".npmrc"), resolve(tmpDirPath, ".npmrc"))
+
+  console.log("Installing all packages in workspaces")
+  await exec("npm", ["install", "--omit=dev"], { cwd: tmpDirPath, stdio: "inherit" })
 
+  const cliPath = resolve(tmpDirPath, workspaces.find((p) => p.name === "@garden-io/cli")!.location)
   // Run pkg and pack up each platform binary
   console.log(chalk.cyan("Packaging garden binaries"))
 
@@ -292,7 +309,7 @@ async function pkgCommon({
   } else {
     const filename = nodeBinaryPlatform === "linuxmusl" ? `node.abi${abi}.musl.node` : `node.abi${abi}.node`
     const abiPath = resolve(
-      repoRoot,
+      GARDEN_CORE_ROOT,
       "node_modules",
       "node-pty-prebuilt-multiarch",
       "prebuilds",
@@ -322,7 +339,7 @@ async function pkgCommon({
       "--output",
       resolve(targetPath, binFilename),
     ],
-    { env: { PKG_CACHE_PATH: pkgFetchTmpDir } }
+    { env: { PKG_CACHE_PATH: pkgFetchTmpDir }, stdio: "inherit" }
   )
 
   console.log(` - ${targetName} -> static`)