Introduce new mirroring and credential settings format and REST API

.gitignore

@@ -135,6 +135,7 @@ typings/
 
 # next.js build output
 .next
+.swc
 
 # macOS folder meta-data
 .DS_Store

client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/ArmeriaCentralDogmaTest.java

@@ -61,7 +61,7 @@ void pushMirrorsJsonFileToMetaRepository() throws UnknownHostException {
                 .build();
 
         final PushResult result = client.forRepo("foo", "meta")
-                                        .commit("summary", Change.ofJsonUpsert("/mirrors.json", "[]"))
+                                        .commit("summary", Change.ofJsonUpsert("/mirrors/foo.json", "{}"))
                                         .push()
                                         .join();
         assertThat(result.revision().major()).isPositive();

common/build.gradle

@@ -7,6 +7,7 @@ dependencies {
     implementation libs.jackson.core
     implementation libs.jackson.databind
     implementation libs.jackson.datatype.jsr310
+    implementation libs.jackson.datatype.jdk8
 
     // Guava
     implementation libs.guava

common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java

@@ -0,0 +1,194 @@
+/*
+ * Copyright 2023 LINE Corporation
+ *
+ * LINE Corporation licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package com.linecorp.centraldogma.internal.api.v1;
+
+import static com.google.common.base.MoreObjects.firstNonNull;
+import static java.util.Objects.requireNonNull;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.MoreObjects;
+
+@JsonInclude(Include.NON_NULL)
+public final class MirrorDto {
+
+    private final String id;
+    private final boolean enabled;
+    private final String projectName;
+    private final String schedule;
+    private final String direction;
+    private final String localRepo;
+    private final String localPath;
+    private final String remoteScheme;
+    private final String remoteUrl;
+    private final String remotePath;
+    private final String remoteBranch;
+    @Nullable
+    private final String gitignore;
+    private final String credentialId;
+
+    @JsonCreator
+    public MirrorDto(@JsonProperty("id") String id,
+                     @JsonProperty("enabled") @Nullable Boolean enabled,
+                     @JsonProperty("projectName") String projectName,
+                     @JsonProperty("schedule") String schedule,
+                     @JsonProperty("direction") String direction,
+                     @JsonProperty("localRepo") String localRepo,
+                     @JsonProperty("localPath") String localPath,
+                     @JsonProperty("remoteScheme") String remoteScheme,
+                     @JsonProperty("remoteUrl") String remoteUrl,
+                     @JsonProperty("remotePath") String remotePath,
+                     @JsonProperty("remoteBranch") String remoteBranch,
+                     @JsonProperty("gitignore") @Nullable String gitignore,
+                     @JsonProperty("credentialId") String credentialId) {
+        this.id = requireNonNull(id, "id");
+        this.enabled = firstNonNull(enabled, true);
+        this.projectName = requireNonNull(projectName, "projectName");
+        this.schedule = requireNonNull(schedule, "schedule");
+        this.direction = requireNonNull(direction, "direction");
+        this.localRepo = requireNonNull(localRepo, "localRepo");
+        this.localPath = requireNonNull(localPath, "localPath");
+        this.remoteScheme = requireNonNull(remoteScheme, "remoteScheme");
+        this.remoteUrl = requireNonNull(remoteUrl, "remoteUrl");
+        this.remotePath = requireNonNull(remotePath, "remotePath");
+        this.remoteBranch = requireNonNull(remoteBranch, "remoteBranch");
+        this.gitignore = gitignore;
+        this.credentialId = requireNonNull(credentialId, "credentialId");
+    }
+
+    @JsonProperty("id")
+    public String id() {
+        return id;
+    }
+
+    @JsonProperty("enabled")
+    public boolean enabled() {
+        return enabled;
+    }
+
+    @JsonProperty("projectName")
+    public String projectName() {
+        return projectName;
+    }
+
+    @JsonProperty("schedule")
+    public String schedule() {
+        return schedule;
+    }
+
+    @JsonProperty("direction")
+    public String direction() {
+        return direction;
+    }
+
+    @JsonProperty("localRepo")
+    public String localRepo() {
+        return localRepo;
+    }
+
+    @JsonProperty("localPath")
+    public String localPath() {
+        return localPath;
+    }
+
+    @JsonProperty("remoteScheme")
+    public String remoteScheme() {
+        return remoteScheme;
+    }
+
+    @JsonProperty("remoteUrl")
+    public String remoteUrl() {
+        return remoteUrl;
+    }
+
+    @JsonProperty("remotePath")
+    public String remotePath() {
+        return remotePath;
+    }
+
+    @JsonProperty("remoteBranch")
+    public String remoteBranch() {
+        return remoteBranch;
+    }
+
+    @Nullable
+    @JsonProperty("gitignore")
+    public String gitignore() {
+        return gitignore;
+    }
+
+    @JsonProperty("credentialId")
+    public String credentialId() {
+        return credentialId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof MirrorDto)) {
+            return false;
+        }
+        final MirrorDto mirrorDto = (MirrorDto) o;
+        return id.equals(mirrorDto.id) &&
+               enabled == mirrorDto.enabled &&
+               projectName.equals(mirrorDto.projectName) &&
+               schedule.equals(mirrorDto.schedule) &&
+               direction.equals(mirrorDto.direction) &&
+               localRepo.equals(mirrorDto.localRepo) &&
+               localPath.equals(mirrorDto.localPath) &&
+               remoteScheme.equals(mirrorDto.remoteScheme) &&
+               remoteUrl.equals(mirrorDto.remoteUrl) &&
+               remotePath.equals(mirrorDto.remotePath) &&
+               remoteBranch.equals(mirrorDto.remoteBranch) &&
+               Objects.equals(gitignore, mirrorDto.gitignore) &&
+               credentialId.equals(mirrorDto.credentialId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, projectName, schedule, direction, localRepo, localPath, remoteScheme,
+                            remoteUrl, remotePath, remoteBranch, gitignore, credentialId, enabled);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+                          .omitNullValues()
+                          .add("id", id)
+                          .add("enabled", enabled)
+                          .add("projectName", projectName)
+                          .add("schedule", schedule)
+                          .add("direction", direction)
+                          .add("localRepo", localRepo)
+                          .add("localPath", localPath)
+                          .add("remoteScheme", remoteScheme)
+                          .add("remoteUrl", remoteUrl)
+                          .add("remotePath", remotePath)
+                          .add("gitignore", gitignore)
+                          .add("credentialId", credentialId)
+                          .toString();
+    }
+}

common/src/main/java/com/linecorp/centraldogma/internal/api/v1/PushResultDto.java

@@ -21,6 +21,7 @@
 
 import java.time.Instant;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -34,9 +35,15 @@ public class PushResultDto {
     private final Revision revision;
     private final String pushedAt;
 
-    public PushResultDto(Revision revision, long commitTimeMillis) {
+    @JsonCreator
+    public PushResultDto(@JsonProperty("revision") Revision revision,
+                         @JsonProperty("pushedAt") Instant pushedAt) {
         this.revision = requireNonNull(revision, "revision");
-        pushedAt = ISO_INSTANT.format(Instant.ofEpochMilli(commitTimeMillis));
+        this.pushedAt = ISO_INSTANT.format(pushedAt);
+    }
+
+    public PushResultDto(Revision revision, long pushAt) {
+        this(revision, Instant.ofEpochMilli(pushAt));
     }
 
     @JsonProperty("revision")

dependencies.toml

@@ -16,6 +16,7 @@ cron-utils = "9.2.0"
 diffutils = "1.3.0"
 docker = "9.4.0"
 download = "5.6.0"
+dropwizard-metrics = "4.2.21"
 eddsa = "0.3.0"
 findbugs = "3.0.2"
 futures-completable = "0.3.6"
@@ -139,6 +140,11 @@ module = "com.googlecode.java-diff-utils:diffutils"
 version.ref = "diffutils"
 relocations = { from = "difflib", to = "com.linecorp.centraldogma.internal.shaded.difflib" }
 
+# Used for testing only.
+[libraries.dropwizard-metrics-core]
+module = "io.dropwizard.metrics:metrics-core"
+version.ref = "dropwizard-metrics"
+
 [libraries.eddsa]
 module = "net.i2p.crypto:eddsa"
 version.ref = "eddsa"

it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ForceRefUpdateTest.java

@@ -104,8 +104,8 @@ void afterEach() {
         dogma.client()
              .forRepo(projName, Project.REPO_META)
              .commit("cleanup",
-                     Change.ofRemoval("/credentials.json"),
-                     Change.ofRemoval("/mirrors.json"))
+                     Change.ofRemoval("/credentials/public-key-id.json"),
+                     Change.ofRemoval("/mirrors/foo.json"))
              .push().join();
     }
 
@@ -200,29 +200,32 @@ private static void assertRevisionAndContent(String expectedRevision,
     private void pushCredentials(String pubKey, String privKey) {
         dogma.client().forRepo(projName, Project.REPO_META)
              .commit("Add a mirror",
-                     Change.ofJsonUpsert("/credentials.json",
-                                         "[{" +
+                     Change.ofJsonUpsert("/credentials/public-key-id.json",
+                                         '{' +
+                                         "  \"id\": \"public-key-id\"," +
                                          "  \"type\": \"public_key\"," +
                                          "  \"hostnamePatterns\": [ \"^.*$\" ]," +
                                          "  \"username\": \"" + "git" + "\"," +
                                          "  \"publicKey\": \"" + pubKey + "\"," +
                                          "  \"privateKey\": \"" + privKey + '"' +
-                                         "}]")
+                                         '}')
              ).push().join();
     }
 
     private void pushMirror(String gitUri, MirrorDirection mirrorDirection) {
         dogma.client().forRepo(projName, Project.REPO_META)
              .commit("Add a mirror",
-                     Change.ofJsonUpsert("/mirrors.json",
-                                         "[{" +
+                     Change.ofJsonUpsert("/mirrors/foo.json",
+                                         '{' +
+                                         "  \"id\": \"foo\"," +
+                                         "  \"enabled\": true," +
                                          "  \"type\": \"single\"," +
                                          "  \"direction\": \"" + mirrorDirection.name() + "\"," +
                                          "  \"localRepo\": \"" + REPO_FOO + "\"," +
                                          "  \"localPath\": \"/\"," +
                                          "  \"remoteUri\": \"" + gitUri + "\"," +
                                          "  \"schedule\": \"0 0 0 1 1 ? 2099\"" +
-                                         "}]"))
+                                         '}'))
              .push().join();
     }
 }

it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorAuthTest.java

@@ -31,8 +31,6 @@
 import org.junit.jupiter.params.provider.MethodSource;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSet.Builder;
 import com.google.common.io.Resources;
@@ -42,6 +40,7 @@
 import com.linecorp.centraldogma.internal.Jackson;
 import com.linecorp.centraldogma.server.CentralDogmaBuilder;
 import com.linecorp.centraldogma.server.MirroringService;
+import com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository;
 import com.linecorp.centraldogma.server.storage.project.Project;
 import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension;
 
@@ -105,19 +104,21 @@ void auth(String projName, String gitUri, JsonNode credential) {
         client.createProject(projName).join();
         client.createRepository(projName, "main").join();
 
-        // Add /credentials.json and /mirrors.json
-        final ArrayNode credentials = JsonNodeFactory.instance.arrayNode().add(credential);
+        // Add /credentials/{id}.json and /mirrors/{id}.json
+        final String credentialId = credential.get("id").asText();
         client.forRepo(projName, Project.REPO_META)
               .commit("Add a mirror",
-                      Change.ofJsonUpsert("/credentials.json", credentials),
-                      Change.ofJsonUpsert("/mirrors.json",
-                                          "[{" +
+                      Change.ofJsonUpsert(DefaultMetaRepository.credentialFile(credentialId), credential),
+                      Change.ofJsonUpsert("/mirrors/main.json",
+                                          '{' +
+                                          "  \"id\": \"main\"," +
+                                          "  \"enabled\": true," +
                                           "  \"type\": \"single\"," +
                                           "  \"direction\": \"REMOTE_TO_LOCAL\"," +
                                           "  \"localRepo\": \"main\"," +
                                           "  \"localPath\": \"/\"," +
                                           "  \"remoteUri\": \"" + gitUri + '"' +
-                                          "}]"))
+                                          '}'))
               .push().join();
 
         // Try to perform mirroring to see if authentication works as expected.
@@ -133,6 +134,8 @@ private static Collection<Arguments> arguments() throws Exception {
                     "git+https://github.com/line/centraldogma-authtest.git",
                     Jackson.readTree(
                             '{' +
+                            "  \"id\": \"password-id\"," +
+                            "  \"enabled\": true," +
                             "  \"type\": \"password\"," +
                             "  \"hostnamePatterns\": [ \"^.*$\" ]," +
                             "  \"username\": \"" + GITHUB_USERNAME + "\"," +
@@ -146,6 +149,8 @@ private static Collection<Arguments> arguments() throws Exception {
                     "git+https://github.com/line/centraldogma-authtest.git",
                     Jackson.readTree(
                             '{' +
+                            "  \"id\": \"access-token-id\"," +
+                            "  \"enabled\": true," +
                             "  \"type\": \"access_token\"," +
                             "  \"hostnamePatterns\": [ \"^.*$\" ]," +
                             "  \"accessToken\": \"" + Jackson.escapeText(GITHUB_ACCESS_TOKEN) + '"' +
@@ -203,6 +208,8 @@ private static void sshAuth(Builder<Arguments> builder, String privateKeyFile, S
                 "git+ssh://github.com/line/centraldogma-authtest.git",
                 Jackson.readTree(
                         '{' +
+                        "  \"id\": \"" + privateKeyFile + "\"," +
+                        "  \"enabled\": true," +
                         "  \"type\": \"public_key\"," +
                         "  \"hostnamePatterns\": [ \"^.*$\" ]," +
                         "  \"username\": \"git\"," +