Use Circle CI dynamic configuration

Now `.circleci/config.continue.yml.j2` is a Jinja2 template that will
get various variables depending on the branch. This allows finer-grained
control on which jobs we run for each type of pipeline.

Author: smola

Diff for: .circleci/config.yml renamed to .circleci/config.continue.yml.j2

@@ -7,11 +7,17 @@ defaults: &defaults
 
 test_matrix: &test_matrix
   parameters:
-    testJvm: [ "ibm8", "semeru8", "zulu8", "oracle8", "11", "zulu11", "17", "ubuntu17" ]
+    testJvm:
+{% for jdk in nocov_jdks %}
+      - "{{ jdk }}"
+{% endfor %}
 
 profiling_test_matrix: &profiling_test_matrix
   parameters:
-    testJvm: [ "8", "zulu8", "oracle8", "11", "zulu11", "17", "ubuntu17" ]
+    testJvm:
+{% for jdk in all_jdks %}
+      - "{{ jdk }}"
+{% endfor %}
 
 system_test_matrix: &system_test_matrix
   parameters:
@@ -179,6 +185,7 @@ commands:
   # 2) Cache keys are prefix matched, and the most recently updated cache that matches will be picked
   #
   # There is a weekly job that runs on Monday mornings that builds a new cache from scratch.
+  {% raw %}
   restore_dependency_cache:
     parameters:
       cacheType:
@@ -239,6 +246,7 @@ commands:
             # Workspace
             - ~/dd-trace-java/.gradle
             - ~/dd-trace-java/workspace
+{% endraw %}
 
   setup_system_tests:
     parameters:
@@ -1048,6 +1056,7 @@ build_test_jobs: &build_test_jobs
       maxWorkers: 4
       testJvm: "8"
 
+{% if flaky %}
   - tests:
       requires:
         - ok_to_test
@@ -1088,6 +1097,7 @@ build_test_jobs: &build_test_jobs
       parallelism: 4
       maxWorkers: 4
       testJvm: "8"
+{% endif %}
 
   - tests:
       requires:
@@ -1128,30 +1138,6 @@ build_test_jobs: &build_test_jobs
       matrix:
         <<: *test_matrix
 
-  - tests:
-      requires:
-        - ok_to_test
-      name: test_semeru11_smoke
-      gradleTarget: "stageMainDist :smokeTest"
-      gradleParameters: "-PskipFlakyTests"
-      stage: smoke
-      cacheType: smoke
-      parallelism: 4
-      maxWorkers: 3
-      testJvm: "semeru11"
-
-  - tests:
-      requires:
-        - ok_to_test
-      name: test_semeru17_smoke
-      gradleTarget: "stageMainDist :smokeTest"
-      gradleParameters: "-PskipFlakyTests"
-      stage: smoke
-      cacheType: smoke
-      parallelism: 4
-      maxWorkers: 3
-      testJvm: "semeru17"
-
   - tests:
       requires:
         - ok_to_test
@@ -1231,24 +1217,18 @@ build_test_jobs: &build_test_jobs
   - fan_in:
       requires:
         - test_published_artifacts
-        - test_8_profiling
-        - test_oracle8_profiling
-        - test_zulu8_profiling
-        - test_zulu11_profiling
-        - test_11_profiling
-        - test_17_profiling
+{% for jdk in all_jdks %}
+        - "test_{{ jdk }}_profiling"
+{% endfor %}
       name: profiling
       stage: profiling
 
   - fan_in:
       requires:
         - test_published_artifacts
-        - test_8_debugger
-        - test_oracle8_debugger
-        - test_zulu8_debugger
-        - test_zulu11_debugger
-        - test_11_debugger
-        - test_17_debugger
+{% for jdk in all_jdks %}
+        - "test_{{ jdk }}_debugger"
+{% endfor %}
       name: debugger
       stage: debugger
 
@@ -1259,13 +1239,9 @@ build_test_jobs: &build_test_jobs
         - check
         - test_published_artifacts
         - agent_integration_tests
-        - test_8
-        - test_ibm8
-        - test_11
-        - test_semeru11_smoke
-        - test_17
-        - test_semeru17_smoke
-        - test_zulu8
+{% for jdk in all_jdks %}
+        - "test_{{ jdk }}"
+{% endfor %}
         - profiling
         - debugger
       name: required

Diff for: .circleci/render_config.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+import os
+import os.path
+import time
+
+import jinja2
+import requests
+
+SCRIPT_DIR = os.path.dirname(__file__)
+
+TPL_FILENAME = "config.continue.yml.j2"
+OUT_FILENAME = "config.continue.yml"
+GENERATED_CONFIG_PATH = os.path.join(SCRIPT_DIR, OUT_FILENAME)
+
+# JDKs that will run on every pipeline.
+ALWAYS_ON_JDKS = {"8", "11", "17"}
+# And these will run only in master and release/ branches.
+MASTER_ONLY_JDKS = {
+    "ibm8",
+    "oracle8",
+    "semeru8",
+    "zulu8",
+    "semeru11",
+    "zulu11",
+    "semeru17",
+    "ubuntu17",
+}
+
+# Get labels from pull requests to override some defaults for jobs to run.
+# `run-tests: all` will run all tests.
+# `run-tests: ibm8` will run the IBM 8 tests.
+# `run-tests: flaky` for flaky tests jobs.
+pr_url = os.environ.get("CIRCLE_PULL_REQUEST")
+if pr_url:
+    pr_num = int(pr_url.split("/")[-1])
+    headers = {}
+    gh_token = os.environ.get("GH_TOKEN")
+    if gh_token:
+        headers["Authorization"] = gh_token
+    else:
+        print("Missing GH_TOKEN, trying anonymously")
+    for _ in range(20):
+        try:
+            resp = requests.get(
+                f"https://api.github.com/repos/DataDog/dd-trace-java/pulls/{pr_num}",
+                timeout=1,
+                headers=headers,
+            )
+            resp.raise_for_status()
+        except Exception as e:
+            print(f"Request filed: {e}")
+            time.sleep(1)
+            continue
+        data = resp.json()
+        break
+
+    labels = data.get("labels", [])
+    labels = [l["name"] for l in labels]
+    labels = {
+        l.replace("run-tests: ", "") for l in labels if l.startswith("run-tests: ")
+    }
+else:
+    labels = set()
+
+
+branch = os.environ.get("CIRCLE_BRANCH", "")
+if branch == "master" or branch.startswith("release/v") or "all" in labels:
+    all_jdks = ALWAYS_ON_JDKS | MASTER_ONLY_JDKS
+else:
+    all_jdks = ALWAYS_ON_JDKS | (MASTER_ONLY_JDKS & labels)
+nocov_jdks = [j for j in all_jdks if j != "8"]
+
+vars = {
+    "all_jdks": all_jdks,
+    "nocov_jdks": nocov_jdks,
+    "flaky": branch == "master" or "flaky" in labels or "all" in labels,
+}
+
+print(f"Variables for this build: {vars}")
+
+loader = jinja2.FileSystemLoader(searchpath=SCRIPT_DIR)
+env = jinja2.Environment(loader=loader)
+tpl = env.get_template(TPL_FILENAME)
+out = tpl.render(**vars)
+
+with open(GENERATED_CONFIG_PATH, "w", encoding="utf-8") as f:
+    f.write(out)

Diff for: .gitignore

@@ -68,6 +68,7 @@ replay_pid*
 ############
 _circle_ci_cache_*
 upstream.env
+/.circleci/config.continue.yml
 
 # Benchmarks #
 benchmark/reports