test: add nuts (#34)

* feat: add nutcase

* fix: add logging for json parsing errors

* chore: rename Nutcase to NutButter

* test: finish up sourcepath, metadata, and manifest deploy tests

* test: still working on deploy nuts

* test: getting nutty

* test: NutButter -> Nutshell

* test: finish deploy tests

* test: split nuts

* test: wip

* test: improve flakiness

* test(wip): retrieve nuts

* test(wip): retrieve packagenames test

* test(wip): add packagenames test

* test: consolidate retrieve seeds

* chore: code review

* test: add convert seed

* test: wip

* test: improve test matrix

* test: add another test for deploy.metadata

* test(wip): stop relying on json

* test(wip): stop relying on push json

* test(wip): stop relying on convert json

* test(wip): stop relying on retrieve json

* test(wip): use SourceMembers for deploy assertions

* test(wip): stop relying on pull json

* test(wip): clean up

* test(wip): more clean up

* test: get dreamhouse-sfdx to pass

* chore: code review

* chore: use new mpd repo

* test: add mpd tests

* test: remove pointless tests

* chore: fix spelling

* chore: bump testkit

* test: fix deploy.testlevel

* test: add --ignoreerrors flag to quick deploy test

Author: mdonnalley

.gitignore

@@ -20,6 +20,7 @@ lib
 *unitcoverage
 .nyc_output
 coverage
+test_session*
 
 # generated docs
 docs
@@ -38,3 +39,6 @@ node_modules
 # os specific files
 .DS_Store
 .idea
+
+# ignore generated nut tests
+test/nuts/generated/

.prettierignore

@@ -0,0 +1,2 @@
+# ignore generated nut tests
+test/nuts/generated/

package.json

@@ -15,16 +15,22 @@
   "devDependencies": {
     "@oclif/dev-cli": "^1",
     "@oclif/plugin-command-snapshot": "^2.0.0",
-    "@salesforce/cli-plugins-testkit": "^0.0.8",
+    "@salesforce/cli-plugins-testkit": "^0.0.15",
     "@salesforce/dev-config": "^2.1.0",
-    "@salesforce/dev-scripts": "^0.7.0",
+    "@salesforce/dev-scripts": "^0.9.1",
+    "@salesforce/plugin-apex": "^0.1.18",
     "@salesforce/plugin-command-reference": "^1.3.0",
+    "@salesforce/plugin-config": "^1.2.6",
     "@salesforce/prettier-config": "^0.0.2",
     "@salesforce/ts-sinon": "1.3.0",
+    "@types/debug": "^4.1.5",
+    "@types/shelljs": "^0.8.8",
     "@typescript-eslint/eslint-plugin": "^4.2.0",
     "@typescript-eslint/parser": "^4.2.0",
     "chai": "^4.2.0",
+    "chai-each": "^0.0.1",
     "cz-conventional-changelog": "^3.3.0",
+    "debug": "^4.3.1",
     "eslint": "^6.8.0",
     "eslint-config-prettier": "^6.11.0",
     "eslint-config-salesforce": "^0.1.0",
@@ -34,12 +40,14 @@
     "eslint-plugin-import": "^2.20.2",
     "eslint-plugin-jsdoc": "^27.0.3",
     "eslint-plugin-prettier": "^3.1.3",
+    "fast-glob": "^3.2.5",
     "husky": "^4.2.5",
     "lint-staged": "^10.2.2",
-    "mocha": "^7.2.0",
+    "mocha": "^8.3.0",
     "nyc": "^15.1.0",
     "prettier": "^2.0.5",
     "pretty-quick": "^2.0.1",
+    "shelljs": "^0.8.4",
     "shx": "0.3.3",
     "sinon": "^9.0.2",
     "ts-node": "^8.10.2",
@@ -71,9 +79,11 @@
     "commands": "./lib/commands",
     "bin": "sfdx",
     "devPlugins": [
-      "@oclif/plugin-help",
       "@oclif/plugin-command-snapshot",
-      "@salesforce/plugin-command-reference"
+      "@oclif/plugin-help",
+      "@salesforce/plugin-command-reference",
+      "@salesforce/plugin-apex",
+      "@salesforce/plugin-config"
     ],
     "topics": {
       "force": {
@@ -90,21 +100,22 @@
   "scripts": {
     "build": "sf-build",
     "clean": "sf-clean",
-    "clean-all": "-clean all",
+    "clean-all": "sf-clean all",
     "clean:lib": "shx rm -rf lib && shx rm -rf coverage && shx rm -rf .nyc_output && shx rm -f oclif.manifest.json",
     "compile": "sf-compile",
     "docs": "sf-docs",
     "format": "sf-format",
     "lint": "sf-lint",
     "postpack": "shx rm -f oclif.manifest.json",
     "posttest": "yarn lint && yarn test:deprecation-policy && yarn test:command-reference",
-    "prepack": "sf-build",
+    "prepack": "sf-prepack",
     "prepare": "sf-install",
     "pretest": "sf-compile-test",
     "test": "sf-test",
     "test:command-reference": "./bin/run commandreference:generate --erroronwarnings",
     "test:deprecation-policy": "./bin/run snapshot:compare",
-    "version": "oclif-dev readme && git add README.md"
+    "test:nuts": "ts-node ./test/nuts/generateNuts.ts && nyc mocha \"**/*.nut.ts\" --slow 3000 --timeout 600000 --parallel --retries 1",
+    "version": "oclif-dev readme"
   },
   "husky": {
     "hooks": {

src/commands/force/source/deploy.ts

@@ -17,7 +17,7 @@ import { SourceCommand } from '../../../sourceCommand';
 Messages.importMessagesDirectory(__dirname);
 const messages = Messages.loadMessages('@salesforce/plugin-source', 'deploy');
 
-export class deploy extends SourceCommand {
+export class Deploy extends SourceCommand {
   public static readonly description = messages.getMessage('description');
   public static readonly examples = messages.getMessage('examples').split(os.EOL);
   public static readonly requiresProject = true;

test/commands/source/deploy.nut.ts

@@ -0,0 +1,24 @@
+# plugin-source NUTs
+
+In testing the source commands, we have 2 significant problems that needs to be solved:
+
+1. How do we test the commands across the old code (force-com-toolbelt) and the new code (plugin-source)?
+2. How do we test the commands across a variety of sfdx projects and metadata?
+
+These problems are solved by generating `.nut.ts` files from `.seed.ts` files using the test matrix exported by `testMatrix.ts`
+
+In practice this means that we generate a `.nut.ts` file for every executable and test repository combination based on the `.seed.ts`. For instance, if we have 5 seeds and if we have 3 test repositories defined in the test matrix, we will generate 30 nuts: 5 seeds * 3 repos * 2 executables (`sfdx` and `bin/run`)
+
+By generating nuts we also maintain the ability the parallelize them since mocha parallelizes tests at the file level. So now we can run those 30 nuts in parallel instead of running them one at a time, which is a huge performance advantage.
+
+However, since we're running the exact same tests on a variety of repositories with different metadata, we have to configure certain aspects so that the tests are meaningful. All the possible configuration is documented in `testMatrix.ts`.
+
+It also means that we're relatively limited in what we can test. For example, it doesn't make sense to write complicated multiple package directory (MPD) tests for repositories that do not have multiple package directories. So that means that the seeds are best used as high-level smoke tests that simply run through the most common use cases and verify json output. **Because of this, we are not locked into to only writing seeds**. It is still entirely possible to write a new `.nut.ts` that lives outside of the test matrix framework.
+
+## Environment Variable Reference
+| Name                       | Description                                               | Default |
+|----------------------------|-----------------------------------------------------------|---------|
+| PLUGIN_SOURCE_SEED_FILTER  | substring of seeds you want to generate                   | NA      |
+| PLUGIN_SOURCE_TEST_BIN_RUN | set to true if you want to include the bin/run executable | false   |
+| PLUGIN_SOURCE_TEST_SFDX    | set to true if you want to include the sfdx executable    | true    |
+