From 531ffc35c6d4d5244647e23408b79b8ee1074b80 Mon Sep 17 00:00:00 2001 From: Ikhun Um Date: Mon, 1 Jul 2024 14:27:59 +0900 Subject: [PATCH] Introduce new mirroring and credential settings format and REST API (#880) Motivation: The ID for mirroring and credential configurations is optional, so I found it difficult to safely update a configuration with REST API. If a user updates a file manually on UI or commit API, the REST API may update a wrong configuration without a unique ID. @trustin suggested changing the directory layout to store a configuration to a file with a unique ID. https://github.com/line/centraldogma/pull/838#pullrequestreview-1410657085 This PR has three major changes: - Migrate the old mirror and credential settings to the new layout. ``` - mirrors - .json - ... - credentials - .json - ... ``` - Add a migration job to automatically migrate the old settings to the new format when a server starts. The old files are renamed by adding `.bak` suffix. e,g. `mirrors.json` -> `mirrors.json.bak`, `credentials.json` -> `credentials.json.bak`. - Add REST API for mirroring and credential configurations. This is a necessary task to add mirror UI. #838 Modifications: - Add `MirroringMigrationService` that is executed when a server starts and scan all `/mirrors.json` and `/credentials.json` in the meta repo of projects and migrate them to the new format. - "id" is a required property in each configuration. Human-readable random words are used to create a unique ID. - Mirror ID format: `mirror---` - Credential ID format: `credential--` - `short_wordlist.txt` is used as the word database. - Change `Mirror` and related classes to have `id`, `enabled` as required fields. - Add `CredentailServiceV1` and `MirrorServiceV1` to serve REST API for CRU. - Create, read, and update operations are implemented in `DefaultMetaRepository`. - Add `RepositoryUri` to represent a repository-specific URI such as a Git repository URL. - Add `MirrorDto` to serialize a mirroring configuration. `Mirror` represents a mirroring task, so `Mirror` is not suitable for serialization. - `MirrorCredential` is used as is instead of creating a new DTO. - Migrated all mirroring tests to use the new configuration format. - Updated site documentation with the new format. Result: - Mirroring and credential settings have been updated to the new formats. - You can now access and modify mirroring and credential resources using the REST API. --- .gitignore | 1 + .../armeria/ArmeriaCentralDogmaTest.java | 2 +- .../internal/api/v1/MirrorDto.java | 194 +++ .../internal/api/v1/PushResultDto.java | 11 +- dependencies.toml | 6 + .../it/mirror/git/ForceRefUpdateTest.java | 19 +- .../it/mirror/git/GitMirrorAuthTest.java | 23 +- ...est.java => GitMirrorIntegrationTest.java} | 12 +- .../git/LegacyGitMirrorSettingsTest.java | 92 ++ .../git/LocalToRemoteGitMirrorTest.java | 12 +- server-mirror-git/build.gradle | 3 + .../internal/mirror/AbstractGitMirror.java | 16 +- .../internal/mirror/DefaultGitMirror.java | 10 +- .../internal/mirror/GitMirrorProvider.java | 16 +- .../server/internal/mirror/SshGitMirror.java | 9 +- .../DefaultMetaRepositoryWithMirrorTest.java | 251 ++-- .../mirror/DefaultMirroringServiceTest.java | 30 +- .../server/internal/mirror/GitMirrorTest.java | 18 +- .../MirroringAndCredentialServiceV1Test.java | 255 ++++ .../MirroringMigrationServiceClusterTest.java | 263 ++++ .../mirror/MirroringMigrationServiceTest.java | 354 +++++ .../internal/mirror/MirroringTestUtils.java | 4 +- .../centraldogma/server/CentralDogma.java | 91 +- .../server/CentralDogmaBuilder.java | 11 +- .../centraldogma/server/PluginGroup.java | 12 +- .../command/StandaloneCommandExecutor.java | 2 +- .../internal/api/AdministrativeService.java | 6 - .../server/internal/api/ContentServiceV1.java | 58 +- .../internal/api/CredentialServiceV1.java | 116 ++ .../internal/api/MirroringServiceV1.java | 147 ++ .../api/auth/RequiresPermissionDecorator.java | 34 +- .../api/auth/RequiresReadPermission.java | 14 + .../api/auth/RequiresWritePermission.java | 14 + .../internal/mirror/AbstractMirror.java | 24 +- .../internal/mirror/CentralDogmaMirror.java | 10 +- .../mirror/DefaultMirroringService.java | 34 +- .../mirror/MirroringMigrationService.java | 464 ++++++ .../server/internal/mirror/MirroringTask.java | 4 +- .../credential/AbstractMirrorCredential.java | 36 +- .../AccessTokenMirrorCredential.java | 11 +- .../credential/NoneMirrorCredential.java | 5 +- .../credential/PasswordMirrorCredential.java | 11 +- .../credential/PublicKeyMirrorCredential.java | 18 +- .../replication/ZooKeeperCommandExecutor.java | 5 +- .../storage/project/DefaultProject.java | 2 +- .../storage/project/ProjectApiManager.java | 2 +- .../CentralDogmaMirrorProvider.java | 22 +- .../repository/DefaultMetaRepository.java | 335 +++-- .../storage/repository/MirrorConfig.java | 32 +- .../{MirrorUtil.java => RepositoryUri.java} | 69 +- .../thrift/CentralDogmaServiceImpl.java | 2 +- .../server/metadata/MetadataService.java | 2 +- .../centraldogma/server/mirror/Mirror.java | 11 +- .../server/mirror/MirrorContext.java | 25 +- .../server/mirror/MirrorCredential.java | 19 +- .../server/plugin/PluginContext.java | 14 +- .../server/plugin/PluginInitContext.java | 6 +- .../project/InternalProjectInitializer.java} | 71 +- .../storage/repository/MetaRepository.java | 47 +- .../server/internal/mirror/short_wordlist.txt | 1296 +++++++++++++++++ .../internal/api/auth/PermissionTest.java | 5 +- .../mirror/CentralDogmaMirrorTest.java | 20 +- .../AccessTokenMirrorCredentialTest.java | 14 +- .../credential/MirrorCredentialTest.java | 17 +- .../credential/NoneMirrorCredentialTest.java | 12 +- .../PasswordMirrorCredentialTest.java | 18 +- .../PublicKeyMirrorCredentialTest.java | 46 +- site/build.gradle | 8 +- site/src/sphinx/mirroring.rst | 155 +- .../internal/ProjectManagerExtension.java | 6 +- .../internal/auth/TestAuthMessageUtil.java | 24 +- .../internal/CentralDogmaRuleDelegate.java | 10 + .../testing/junit/CentralDogmaExtension.java | 10 + .../xds/internal/ControlPlanePlugin.java | 12 +- .../xds/internal/XdsTestUtil.java | 2 +- 75 files changed, 4477 insertions(+), 565 deletions(-) create mode 100644 common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java rename it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/{GitMirrorTest.java => GitMirrorIntegrationTest.java} (98%) create mode 100644 it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LegacyGitMirrorSettingsTest.java create mode 100644 server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringAndCredentialServiceV1Test.java create mode 100644 server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationServiceClusterTest.java create mode 100644 server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationServiceTest.java create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/internal/api/CredentialServiceV1.java create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java create mode 100644 server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationService.java rename server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/{MirrorUtil.java => RepositoryUri.java} (65%) rename server/src/main/java/com/linecorp/centraldogma/server/{internal/storage/project/ProjectInitializer.java => storage/project/InternalProjectInitializer.java} (69%) create mode 100644 server/src/main/resources/com/linecorp/centraldogma/server/internal/mirror/short_wordlist.txt diff --git a/.gitignore b/.gitignore index a224633437..b163fff60a 100644 --- a/.gitignore +++ b/.gitignore @@ -135,6 +135,7 @@ typings/ # next.js build output .next +.swc # macOS folder meta-data .DS_Store diff --git a/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/ArmeriaCentralDogmaTest.java b/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/ArmeriaCentralDogmaTest.java index ec7b1c9ed8..0d268c56fc 100644 --- a/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/ArmeriaCentralDogmaTest.java +++ b/client/java-armeria/src/test/java/com/linecorp/centraldogma/client/armeria/ArmeriaCentralDogmaTest.java @@ -61,7 +61,7 @@ void pushMirrorsJsonFileToMetaRepository() throws UnknownHostException { .build(); final PushResult result = client.forRepo("foo", "meta") - .commit("summary", Change.ofJsonUpsert("/mirrors.json", "[]")) + .commit("summary", Change.ofJsonUpsert("/mirrors/foo.json", "{}")) .push() .join(); assertThat(result.revision().major()).isPositive(); diff --git a/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java b/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java new file mode 100644 index 0000000000..92c1eccca7 --- /dev/null +++ b/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/MirrorDto.java @@ -0,0 +1,194 @@ +/* + * Copyright 2023 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + */ + +package com.linecorp.centraldogma.internal.api.v1; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static java.util.Objects.requireNonNull; + +import java.util.Objects; + +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; + +@JsonInclude(Include.NON_NULL) +public final class MirrorDto { + + private final String id; + private final boolean enabled; + private final String projectName; + private final String schedule; + private final String direction; + private final String localRepo; + private final String localPath; + private final String remoteScheme; + private final String remoteUrl; + private final String remotePath; + private final String remoteBranch; + @Nullable + private final String gitignore; + private final String credentialId; + + @JsonCreator + public MirrorDto(@JsonProperty("id") String id, + @JsonProperty("enabled") @Nullable Boolean enabled, + @JsonProperty("projectName") String projectName, + @JsonProperty("schedule") String schedule, + @JsonProperty("direction") String direction, + @JsonProperty("localRepo") String localRepo, + @JsonProperty("localPath") String localPath, + @JsonProperty("remoteScheme") String remoteScheme, + @JsonProperty("remoteUrl") String remoteUrl, + @JsonProperty("remotePath") String remotePath, + @JsonProperty("remoteBranch") String remoteBranch, + @JsonProperty("gitignore") @Nullable String gitignore, + @JsonProperty("credentialId") String credentialId) { + this.id = requireNonNull(id, "id"); + this.enabled = firstNonNull(enabled, true); + this.projectName = requireNonNull(projectName, "projectName"); + this.schedule = requireNonNull(schedule, "schedule"); + this.direction = requireNonNull(direction, "direction"); + this.localRepo = requireNonNull(localRepo, "localRepo"); + this.localPath = requireNonNull(localPath, "localPath"); + this.remoteScheme = requireNonNull(remoteScheme, "remoteScheme"); + this.remoteUrl = requireNonNull(remoteUrl, "remoteUrl"); + this.remotePath = requireNonNull(remotePath, "remotePath"); + this.remoteBranch = requireNonNull(remoteBranch, "remoteBranch"); + this.gitignore = gitignore; + this.credentialId = requireNonNull(credentialId, "credentialId"); + } + + @JsonProperty("id") + public String id() { + return id; + } + + @JsonProperty("enabled") + public boolean enabled() { + return enabled; + } + + @JsonProperty("projectName") + public String projectName() { + return projectName; + } + + @JsonProperty("schedule") + public String schedule() { + return schedule; + } + + @JsonProperty("direction") + public String direction() { + return direction; + } + + @JsonProperty("localRepo") + public String localRepo() { + return localRepo; + } + + @JsonProperty("localPath") + public String localPath() { + return localPath; + } + + @JsonProperty("remoteScheme") + public String remoteScheme() { + return remoteScheme; + } + + @JsonProperty("remoteUrl") + public String remoteUrl() { + return remoteUrl; + } + + @JsonProperty("remotePath") + public String remotePath() { + return remotePath; + } + + @JsonProperty("remoteBranch") + public String remoteBranch() { + return remoteBranch; + } + + @Nullable + @JsonProperty("gitignore") + public String gitignore() { + return gitignore; + } + + @JsonProperty("credentialId") + public String credentialId() { + return credentialId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MirrorDto)) { + return false; + } + final MirrorDto mirrorDto = (MirrorDto) o; + return id.equals(mirrorDto.id) && + enabled == mirrorDto.enabled && + projectName.equals(mirrorDto.projectName) && + schedule.equals(mirrorDto.schedule) && + direction.equals(mirrorDto.direction) && + localRepo.equals(mirrorDto.localRepo) && + localPath.equals(mirrorDto.localPath) && + remoteScheme.equals(mirrorDto.remoteScheme) && + remoteUrl.equals(mirrorDto.remoteUrl) && + remotePath.equals(mirrorDto.remotePath) && + remoteBranch.equals(mirrorDto.remoteBranch) && + Objects.equals(gitignore, mirrorDto.gitignore) && + credentialId.equals(mirrorDto.credentialId); + } + + @Override + public int hashCode() { + return Objects.hash(id, projectName, schedule, direction, localRepo, localPath, remoteScheme, + remoteUrl, remotePath, remoteBranch, gitignore, credentialId, enabled); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .omitNullValues() + .add("id", id) + .add("enabled", enabled) + .add("projectName", projectName) + .add("schedule", schedule) + .add("direction", direction) + .add("localRepo", localRepo) + .add("localPath", localPath) + .add("remoteScheme", remoteScheme) + .add("remoteUrl", remoteUrl) + .add("remotePath", remotePath) + .add("gitignore", gitignore) + .add("credentialId", credentialId) + .toString(); + } +} diff --git a/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/PushResultDto.java b/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/PushResultDto.java index 41cb884890..a20748b52a 100644 --- a/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/PushResultDto.java +++ b/common/src/main/java/com/linecorp/centraldogma/internal/api/v1/PushResultDto.java @@ -21,6 +21,7 @@ import java.time.Instant; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; @@ -34,9 +35,15 @@ public class PushResultDto { private final Revision revision; private final String pushedAt; - public PushResultDto(Revision revision, long commitTimeMillis) { + @JsonCreator + public PushResultDto(@JsonProperty("revision") Revision revision, + @JsonProperty("pushedAt") Instant pushedAt) { this.revision = requireNonNull(revision, "revision"); - pushedAt = ISO_INSTANT.format(Instant.ofEpochMilli(commitTimeMillis)); + this.pushedAt = ISO_INSTANT.format(pushedAt); + } + + public PushResultDto(Revision revision, long pushAt) { + this(revision, Instant.ofEpochMilli(pushAt)); } @JsonProperty("revision") diff --git a/dependencies.toml b/dependencies.toml index 9630c7bee6..5f3f33f2e7 100644 --- a/dependencies.toml +++ b/dependencies.toml @@ -16,6 +16,7 @@ cron-utils = "9.2.0" diffutils = "1.3.0" docker = "9.4.0" download = "5.6.0" +dropwizard-metrics = "4.2.21" eddsa = "0.3.0" findbugs = "3.0.2" futures-completable = "0.3.6" @@ -139,6 +140,11 @@ module = "com.googlecode.java-diff-utils:diffutils" version.ref = "diffutils" relocations = { from = "difflib", to = "com.linecorp.centraldogma.internal.shaded.difflib" } +# Used for testing only. +[libraries.dropwizard-metrics-core] +module = "io.dropwizard.metrics:metrics-core" +version.ref = "dropwizard-metrics" + [libraries.eddsa] module = "net.i2p.crypto:eddsa" version.ref = "eddsa" diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ForceRefUpdateTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ForceRefUpdateTest.java index 91d3956cfb..6c5d03aeb5 100644 --- a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ForceRefUpdateTest.java +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/ForceRefUpdateTest.java @@ -104,8 +104,8 @@ void afterEach() { dogma.client() .forRepo(projName, Project.REPO_META) .commit("cleanup", - Change.ofRemoval("/credentials.json"), - Change.ofRemoval("/mirrors.json")) + Change.ofRemoval("/credentials/public-key-id.json"), + Change.ofRemoval("/mirrors/foo.json")) .push().join(); } @@ -200,29 +200,32 @@ private static void assertRevisionAndContent(String expectedRevision, private void pushCredentials(String pubKey, String privKey) { dogma.client().forRepo(projName, Project.REPO_META) .commit("Add a mirror", - Change.ofJsonUpsert("/credentials.json", - "[{" + + Change.ofJsonUpsert("/credentials/public-key-id.json", + '{' + + " \"id\": \"public-key-id\"," + " \"type\": \"public_key\"," + " \"hostnamePatterns\": [ \"^.*$\" ]," + " \"username\": \"" + "git" + "\"," + " \"publicKey\": \"" + pubKey + "\"," + " \"privateKey\": \"" + privKey + '"' + - "}]") + '}') ).push().join(); } private void pushMirror(String gitUri, MirrorDirection mirrorDirection) { dogma.client().forRepo(projName, Project.REPO_META) .commit("Add a mirror", - Change.ofJsonUpsert("/mirrors.json", - "[{" + + Change.ofJsonUpsert("/mirrors/foo.json", + '{' + + " \"id\": \"foo\"," + + " \"enabled\": true," + " \"type\": \"single\"," + " \"direction\": \"" + mirrorDirection.name() + "\"," + " \"localRepo\": \"" + REPO_FOO + "\"," + " \"localPath\": \"/\"," + " \"remoteUri\": \"" + gitUri + "\"," + " \"schedule\": \"0 0 0 1 1 ? 2099\"" + - "}]")) + '}')) .push().join(); } } diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorAuthTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorAuthTest.java index fd4d3692c1..cd58d9e977 100644 --- a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorAuthTest.java +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorAuthTest.java @@ -31,8 +31,6 @@ import org.junit.jupiter.params.provider.MethodSource; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet.Builder; import com.google.common.io.Resources; @@ -42,6 +40,7 @@ import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.server.CentralDogmaBuilder; import com.linecorp.centraldogma.server.MirroringService; +import com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; @@ -105,19 +104,21 @@ void auth(String projName, String gitUri, JsonNode credential) { client.createProject(projName).join(); client.createRepository(projName, "main").join(); - // Add /credentials.json and /mirrors.json - final ArrayNode credentials = JsonNodeFactory.instance.arrayNode().add(credential); + // Add /credentials/{id}.json and /mirrors/{id}.json + final String credentialId = credential.get("id").asText(); client.forRepo(projName, Project.REPO_META) .commit("Add a mirror", - Change.ofJsonUpsert("/credentials.json", credentials), - Change.ofJsonUpsert("/mirrors.json", - "[{" + + Change.ofJsonUpsert(DefaultMetaRepository.credentialFile(credentialId), credential), + Change.ofJsonUpsert("/mirrors/main.json", + '{' + + " \"id\": \"main\"," + + " \"enabled\": true," + " \"type\": \"single\"," + " \"direction\": \"REMOTE_TO_LOCAL\"," + " \"localRepo\": \"main\"," + " \"localPath\": \"/\"," + " \"remoteUri\": \"" + gitUri + '"' + - "}]")) + '}')) .push().join(); // Try to perform mirroring to see if authentication works as expected. @@ -133,6 +134,8 @@ private static Collection arguments() throws Exception { "git+https://github.com/line/centraldogma-authtest.git", Jackson.readTree( '{' + + " \"id\": \"password-id\"," + + " \"enabled\": true," + " \"type\": \"password\"," + " \"hostnamePatterns\": [ \"^.*$\" ]," + " \"username\": \"" + GITHUB_USERNAME + "\"," + @@ -146,6 +149,8 @@ private static Collection arguments() throws Exception { "git+https://github.com/line/centraldogma-authtest.git", Jackson.readTree( '{' + + " \"id\": \"access-token-id\"," + + " \"enabled\": true," + " \"type\": \"access_token\"," + " \"hostnamePatterns\": [ \"^.*$\" ]," + " \"accessToken\": \"" + Jackson.escapeText(GITHUB_ACCESS_TOKEN) + '"' + @@ -203,6 +208,8 @@ private static void sshAuth(Builder builder, String privateKeyFile, S "git+ssh://github.com/line/centraldogma-authtest.git", Jackson.readTree( '{' + + " \"id\": \"" + privateKeyFile + "\"," + + " \"enabled\": true," + " \"type\": \"public_key\"," + " \"hostnamePatterns\": [ \"^.*$\" ]," + " \"username\": \"git\"," + diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorIntegrationTest.java similarity index 98% rename from it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java rename to it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorIntegrationTest.java index a3a2b4f441..e943c507f0 100644 --- a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorTest.java +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/GitMirrorIntegrationTest.java @@ -74,7 +74,7 @@ import com.linecorp.centraldogma.testing.internal.TestUtil; import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; -class GitMirrorTest { +class GitMirrorIntegrationTest { private static final int MAX_NUM_FILES = 32; private static final long MAX_NUM_BYTES = 1048576; // 1 MiB @@ -480,9 +480,11 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N final String localPath0 = localPath == null ? "/" : localPath; final String remoteUri = gitUri + firstNonNull(remotePath, ""); client.forRepo(projName, Project.REPO_META) - .commit("Add /mirrors.json", - Change.ofJsonUpsert("/mirrors.json", - "[{" + + .commit("Add /mirrors/foo.json", + Change.ofJsonUpsert("/mirrors/foo.json", + '{' + + " \"id\": \"foo\"," + + " \"enabled\": true," + " \"type\": \"single\"," + " \"direction\": \"REMOTE_TO_LOCAL\"," + " \"localRepo\": \"" + localRepo + "\"," + @@ -490,7 +492,7 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N " \"remoteUri\": \"" + remoteUri + "\"," + " \"schedule\": \"0 0 0 1 1 ? 2099\"," + " \"gitignore\": " + firstNonNull(gitignore, "\"\"") + - "}]")) + '}')) .push().join(); } diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LegacyGitMirrorSettingsTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LegacyGitMirrorSettingsTest.java new file mode 100644 index 0000000000..963ec8b992 --- /dev/null +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LegacyGitMirrorSettingsTest.java @@ -0,0 +1,92 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.it.mirror.git; + +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.getAccessToken; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.concurrent.CompletionException; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.linecorp.centraldogma.client.CentralDogma; +import com.linecorp.centraldogma.client.armeria.ArmeriaCentralDogmaBuilder; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.InvalidPushException; +import com.linecorp.centraldogma.server.CentralDogmaBuilder; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil; +import com.linecorp.centraldogma.testing.internal.auth.TestAuthProviderFactory; +import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; + +class LegacyGitMirrorSettingsTest { + + @RegisterExtension + static final CentralDogmaExtension dogma = new CentralDogmaExtension() { + @Override + protected void configure(CentralDogmaBuilder builder) { + builder.authProviderFactory(new TestAuthProviderFactory()); + builder.administrators(TestAuthMessageUtil.USERNAME); + } + }; + + @Test + void shouldNotAllowLegacyMirrorSettings() throws Exception { + final String accessToken = getAccessToken(dogma.httpClient(), + TestAuthMessageUtil.USERNAME2, + TestAuthMessageUtil.PASSWORD2); + final CentralDogma client = new ArmeriaCentralDogmaBuilder() + .accessToken(accessToken) + .host("127.0.0.1", dogma.serverAddress().getPort()) + .build(); + + client.createProject("foo").join(); + client.createRepository("foo", "bar").join(); + + assertThatThrownBy(() -> { + client.forRepo("foo", Project.REPO_META) + .commit("Add /mirrors.json", + Change.ofJsonUpsert("/mirrors.json", + "[{" + + " \"id\": \"foo\"," + + " \"enabled\": true," + + " \"type\": \"single\"," + + " \"direction\": \"REMOTE_TO_LOCAL\"," + + " \"localRepo\": \"local\"," + + " \"localPath\": \"localPath0\"," + + " \"remoteUri\": \"remoteUri\"," + + " \"schedule\": \"0 0 0 1 1 ? 2099\"" + + "}]")) + .push().join(); + }).isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(InvalidPushException.class) + .hasMessageContaining("'/mirrors.json' file is not allowed to create."); + + assertThatThrownBy(() -> { + client.forRepo("foo", Project.REPO_META) + .commit("Add /credentials.json", + Change.ofJsonUpsert("/credentials.json", + "[{" + + " \"id\": \"access-token-id\"" + + "}]")) + .push().join(); + }).isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(InvalidPushException.class) + .hasMessageContaining("'/credentials.json' file is not allowed to create."); + } +} diff --git a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java index d9c9982aae..df7c587ec3 100644 --- a/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java +++ b/it/mirror/src/test/java/com/linecorp/centraldogma/it/mirror/git/LocalToRemoteGitMirrorTest.java @@ -17,7 +17,7 @@ package com.linecorp.centraldogma.it.mirror.git; import static com.google.common.base.MoreObjects.firstNonNull; -import static com.linecorp.centraldogma.it.mirror.git.GitMirrorTest.addToGitIndex; +import static com.linecorp.centraldogma.it.mirror.git.GitMirrorIntegrationTest.addToGitIndex; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.eclipse.jgit.lib.Constants.R_HEADS; @@ -359,9 +359,11 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N final String localPath0 = localPath == null ? "/" : localPath; final String remoteUri = gitUri + firstNonNull(remotePath, ""); client.forRepo(projName, Project.REPO_META) - .commit("Add /mirrors.json", - Change.ofJsonUpsert("/mirrors.json", - "[{" + + .commit("Add /mirrors/foo.json", + Change.ofJsonUpsert("/mirrors/foo.json", + '{' + + " \"id\": \"foo\"," + + " \"enabled\": true," + " \"type\": \"single\"," + " \"direction\": \"" + direction + "\"," + " \"localRepo\": \"" + localRepo + "\"," + @@ -369,7 +371,7 @@ private void pushMirrorSettings(String localRepo, @Nullable String localPath, @N " \"remoteUri\": \"" + remoteUri + "\"," + " \"schedule\": \"0 0 0 1 1 ? 2099\"," + " \"gitignore\": " + firstNonNull(gitignore, "\"\"") + - "}]")) + '}')) .push().join(); } diff --git a/server-mirror-git/build.gradle b/server-mirror-git/build.gradle index c05ec9c114..ec30fd5132 100644 --- a/server-mirror-git/build.gradle +++ b/server-mirror-git/build.gradle @@ -8,4 +8,7 @@ dependencies { implementation libs.jgit6 implementation libs.mina.sshd.core implementation libs.mina.sshd.git + + testImplementation libs.zookeeper + testImplementation libs.dropwizard.metrics.core } diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java index 9ccf4840c4..d64d72f0cf 100644 --- a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractGitMirror.java @@ -115,12 +115,12 @@ abstract class AbstractGitMirror extends AbstractMirror { @Nullable private IgnoreNode ignoreNode; - AbstractGitMirror(Cron schedule, MirrorDirection direction, MirrorCredential credential, - Repository localRepo, String localPath, - URI remoteRepoUri, String remotePath, @Nullable String remoteBranch, + AbstractGitMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction, + MirrorCredential credential, Repository localRepo, String localPath, + URI remoteRepoUri, String remotePath, String remoteBranch, @Nullable String gitignore) { - super(schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, remoteBranch, - gitignore); + super(id, enabled, schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, + remoteBranch, gitignore); if (gitignore != null) { ignoreNode = new IgnoreNode(); @@ -272,7 +272,7 @@ void mirrorRemoteToLocal( if (ignoreNode != null && path.startsWith(remotePath())) { assert ignoreNode != null; if (ignoreNode.isIgnored('/' + path.substring(remotePath().length()), - fileMode == FileMode.TREE) == MatchResult.IGNORED) { + fileMode == FileMode.TREE) == MatchResult.IGNORED) { continue; } } @@ -364,7 +364,7 @@ private boolean needsFetch(Ref headBranchRef, String mirrorStatePath, Revision l } private Ref getHeadBranchRef(GitWithAuth git) throws GitAPIException { - if (remoteBranch() != null) { + if (!remoteBranch().isEmpty()) { final String headBranchRefName = Constants.R_HEADS + remoteBranch(); final Collection refs = lsRemote(git, true); return findHeadBranchRef(git, headBranchRefName, refs); @@ -390,7 +390,7 @@ private Ref getHeadBranchRef(GitWithAuth git) throws GitAPIException { } private static Collection lsRemote(GitWithAuth git, - boolean setHeads) throws GitAPIException { + boolean setHeads) throws GitAPIException { return git.lsRemote() .setTags(false) .setTimeout(GIT_TIMEOUT_SECS) diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java index 1e5d88e6c3..9e171b5bae 100644 --- a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultGitMirror.java @@ -42,12 +42,12 @@ final class DefaultGitMirror extends AbstractGitMirror { private static final Consumer> NOOP_CONFIGURATOR = command -> {}; - DefaultGitMirror(Cron schedule, MirrorDirection direction, MirrorCredential credential, - Repository localRepo, String localPath, - URI remoteRepoUri, String remotePath, @Nullable String remoteBranch, + DefaultGitMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction, + MirrorCredential credential, Repository localRepo, String localPath, + URI remoteRepoUri, String remotePath, String remoteBranch, @Nullable String gitignore) { - super(schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, remoteBranch, - gitignore); + super(id, enabled, schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, + remoteBranch, gitignore); } @Override diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorProvider.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorProvider.java index 98fffb8959..dd6a2baee8 100644 --- a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorProvider.java +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorProvider.java @@ -16,7 +16,6 @@ package com.linecorp.centraldogma.server.internal.mirror; -import static com.linecorp.centraldogma.server.internal.storage.repository.MirrorUtil.split; import static com.linecorp.centraldogma.server.mirror.MirrorSchemes.SCHEME_GIT; import static com.linecorp.centraldogma.server.mirror.MirrorSchemes.SCHEME_GIT_FILE; import static com.linecorp.centraldogma.server.mirror.MirrorSchemes.SCHEME_GIT_HTTP; @@ -26,6 +25,7 @@ import java.net.URI; +import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryUri; import com.linecorp.centraldogma.server.mirror.Mirror; import com.linecorp.centraldogma.server.mirror.MirrorContext; import com.linecorp.centraldogma.server.mirror.MirrorProvider; @@ -44,20 +44,22 @@ public Mirror newMirror(MirrorContext context) { switch (scheme) { case SCHEME_GIT_SSH: { - final String[] components = split(remoteUri, "git"); - return new SshGitMirror(context.schedule(), context.direction(), context.credential(), + final RepositoryUri repositoryUri = RepositoryUri.parse(remoteUri, "git"); + return new SshGitMirror(context.id(), context.enabled(), context.schedule(), + context.direction(), context.credential(), context.localRepo(), context.localPath(), - URI.create(components[0]), components[1], components[2], + repositoryUri.uri(), repositoryUri.path(), repositoryUri.branch(), context.gitignore()); } case SCHEME_GIT_HTTP: case SCHEME_GIT_HTTPS: case SCHEME_GIT: case SCHEME_GIT_FILE: { - final String[] components = split(remoteUri, "git"); - return new DefaultGitMirror(context.schedule(), context.direction(), context.credential(), + final RepositoryUri repositoryUri = RepositoryUri.parse(remoteUri, "git"); + return new DefaultGitMirror(context.id(), context.enabled(), context.schedule(), + context.direction(), context.credential(), context.localRepo(), context.localPath(), - URI.create(components[0]), components[1], components[2], + repositoryUri.uri(), repositoryUri.path(), repositoryUri.branch(), context.gitignore()); } } diff --git a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java index 76f90376b0..ded2eb0ede 100644 --- a/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java +++ b/server-mirror-git/src/main/java/com/linecorp/centraldogma/server/internal/mirror/SshGitMirror.java @@ -79,11 +79,12 @@ final class SshGitMirror extends AbstractGitMirror { // We might create multiple BouncyCastleRandom later and poll them, if necessary. private static final BouncyCastleRandom bounceCastleRandom = new BouncyCastleRandom(); - SshGitMirror(Cron schedule, MirrorDirection direction, MirrorCredential credential, - Repository localRepo, String localPath, - URI remoteRepoUri, String remotePath, @Nullable String remoteBranch, + SshGitMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction, + MirrorCredential credential, Repository localRepo, String localPath, + URI remoteRepoUri, String remotePath, String remoteBranch, @Nullable String gitignore) { - super(schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, remoteBranch, + super(id, enabled, schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, + remoteBranch, gitignore); } diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java index 4e1fef0a77..596043926b 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMetaRepositoryWithMirrorTest.java @@ -16,62 +16,80 @@ package com.linecorp.centraldogma.server.internal.mirror; -import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.PATH_CREDENTIALS; -import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.PATH_MIRRORS; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.linecorp.centraldogma.server.internal.storage.repository.MirrorConfig.DEFAULT_SCHEDULE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.File; import java.util.Comparator; import java.util.List; -import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.CompletionException; +import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import com.cronutils.model.CronType; import com.cronutils.model.definition.CronDefinitionBuilder; import com.cronutils.parser.CronParser; -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.collect.ImmutableList; -import com.linecorp.armeria.common.metric.NoopMeterRegistry; import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.common.ShuttingDownException; +import com.linecorp.centraldogma.internal.api.v1.MirrorDto; +import com.linecorp.centraldogma.server.command.Command; +import com.linecorp.centraldogma.server.command.CommandExecutor; +import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.internal.mirror.credential.NoneMirrorCredential; import com.linecorp.centraldogma.server.internal.mirror.credential.PasswordMirrorCredential; -import com.linecorp.centraldogma.server.internal.storage.project.DefaultProjectManager; import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryMetadataException; import com.linecorp.centraldogma.server.mirror.Mirror; +import com.linecorp.centraldogma.server.mirror.MirrorCredential; import com.linecorp.centraldogma.server.mirror.MirrorDirection; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import com.linecorp.centraldogma.server.storage.repository.MetaRepository; +import com.linecorp.centraldogma.testing.internal.ProjectManagerExtension; import com.linecorp.centraldogma.testing.internal.TestUtil; class DefaultMetaRepositoryWithMirrorTest { - private static final Change UPSERT_CREDENTIALS = Change.ofJsonUpsert( - PATH_CREDENTIALS, - "[{" + - " \"type\": \"password\"," + - " \"id\": \"alice\"," + - " \"hostnamePatterns\": [ \"^foo\\\\.com$\" ]," + - " \"username\": \"alice\"," + - " \"password\": \"secret_a\"" + - "},{" + - " \"type\": \"password\"," + - " \"hostnamePatterns\": [ \"^.*\\\\.com$\" ]," + - " \"username\": \"bob\"," + - " \"password\": \"secret_b\"" + - "}]"); + private static final List> UPSERT_RAW_CREDENTIALS = ImmutableList.of( + Change.ofJsonUpsert( + "/credentials/alice.json", + '{' + + " \"id\": \"alice\"," + + " \"type\": \"password\"," + + " \"hostnamePatterns\": [ \"^foo\\\\.com$\" ]," + + " \"username\": \"alice\"," + + " \"password\": \"secret_a\"" + + '}'), + Change.ofJsonUpsert( + "/credentials/bob.json", + '{' + + " \"id\": \"bob\"," + + " \"type\": \"password\"," + + " \"hostnamePatterns\": [ \"^.*\\\\.com$\" ]," + + " \"username\": \"bob\"," + + " \"password\": \"secret_b\"" + + '}')); + + private static final List CREDENTIALS = ImmutableList.of( + new PasswordMirrorCredential( + "alice", true, ImmutableList.of(Pattern.compile("^foo\\.com$")), + "alice", "secret_a"), + new PasswordMirrorCredential( + "bob", true, ImmutableList.of(Pattern.compile("^.*\\.com$")), + "bob", "secret_b")); private static final CronParser cronParser = new CronParser( CronDefinitionBuilder.instanceDefinitionFor(CronType.QUARTZ)); @@ -81,18 +99,15 @@ class DefaultMetaRepositoryWithMirrorTest { private static ProjectManager pm; + @RegisterExtension + static final ProjectManagerExtension pmExtension = new ProjectManagerExtension(); + private Project project; private MetaRepository metaRepo; @BeforeAll static void init() { - pm = new DefaultProjectManager(rootDir, ForkJoinPool.commonPool(), - MoreExecutors.directExecutor(), NoopMeterRegistry.get(), null); - } - - @AfterAll - static void destroy() { - pm.close(ShuttingDownException::new); + pm = pmExtension.projectManager(); } @BeforeEach @@ -104,16 +119,8 @@ void setUp(TestInfo testInfo) { @Test void testEmptyMirrors() { - // should return an empty result when both /credentials.json and /mirrors.json are non-existent. - assertThat(metaRepo.mirrors()).isEmpty(); - - // should return an empty result when /credentials.json exists and /mirrors.json does not. - metaRepo.commit(Revision.HEAD, 0, Author.SYSTEM, "", Change.ofJsonUpsert("/credentials.json", "[]")); - assertThat(metaRepo.mirrors()).isEmpty(); - - // should return an empty result when both /credentials.json and /mirrors.json exist. - metaRepo.commit(Revision.HEAD, 0, Author.SYSTEM, "", Change.ofJsonUpsert("/mirrors.json", "[]")); - assertThat(metaRepo.mirrors()).isEmpty(); + // should return an empty result when both /credentials/ and /mirrors/ are non-existent. + assertThat(metaRepo.mirrors().join()).isEmpty(); } /** @@ -121,53 +128,102 @@ void testEmptyMirrors() { */ @Test void testInvalidMirrors() { - // not an array but an object + // not an object but an array metaRepo.commit(Revision.HEAD, 0, Author.SYSTEM, "", - Change.ofJsonUpsert(PATH_MIRRORS, "{}")).join(); - assertThatThrownBy(() -> metaRepo.mirrors()).isInstanceOf(RepositoryMetadataException.class); + Change.ofJsonUpsert("/mirrors/foo.json", "[]")).join(); + assertThatThrownBy(() -> metaRepo.mirror("foo").join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(RepositoryMetadataException.class); - // not an array but a value + // not an object but a value metaRepo.commit(Revision.HEAD, 0, Author.SYSTEM, "", - Change.ofJsonUpsert(PATH_MIRRORS, "\"oops\"")).join(); - assertThatThrownBy(() -> metaRepo.mirrors()).isInstanceOf(RepositoryMetadataException.class); + Change.ofJsonUpsert("/mirrors/bar.json", "\"oops\"")).join(); + assertThatThrownBy(() -> metaRepo.mirror("bar").join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(RepositoryMetadataException.class); - // an array that contains null. + // an empty object metaRepo.commit(Revision.HEAD, 0, Author.SYSTEM, "", - Change.ofJsonUpsert(PATH_MIRRORS, "[ null ]")).join(); - assertThatThrownBy(() -> metaRepo.mirrors()).isInstanceOf(RepositoryMetadataException.class); + Change.ofJsonUpsert("/mirrors/qux.json", "{}")).join(); + assertThatThrownBy(() -> metaRepo.mirror("qux").join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(RepositoryMetadataException.class); } - @Test - void testMirror() { - metaRepo.commit(Revision.HEAD, 0, Author.SYSTEM, "", - Change.ofJsonUpsert( - PATH_MIRRORS, - "[{" + - " \"enabled\": true," + - " \"direction\": \"LOCAL_TO_REMOTE\"," + - " \"localRepo\": \"foo\"," + - " \"localPath\": \"/mirrors/foo\"," + - " \"remoteUri\": \"git+ssh://foo.com/foo.git\"" + - "},{" + - " \"enabled\": true," + - " \"schedule\": \"*/10 * * * * ?\"," + - " \"direction\": \"REMOTE_TO_LOCAL\"," + - " \"localRepo\": \"bar\"," + - " \"remoteUri\": \"git+ssh://bar.com/bar.git/some-path\"" + - "}, {" + - " \"direction\": \"LOCAL_TO_REMOTE\"," + - " \"localRepo\": \"qux\"," + - " \"remoteUri\": \"git+ssh://qux.net/qux.git#develop\"" + - "}, {" + - " \"enabled\": false," + // Disabled - " \"direction\": \"LOCAL_TO_REMOTE\"," + - " \"localRepo\": \"foo\"," + - " \"localPath\": \"/mirrors/bar\"," + - " \"remoteUri\": \"git+ssh://bar.com/bar.git\"" + - "}]"), UPSERT_CREDENTIALS).join(); + @ValueSource(booleans = { true, false }) + @ParameterizedTest + void testMirror(boolean useRawApi) { + if (useRawApi) { + final List> mirrors = ImmutableList.of( + Change.ofJsonUpsert( + "/mirrors/foo.json", + '{' + + " \"id\": \"foo\"," + + " \"enabled\": true," + + " \"direction\": \"LOCAL_TO_REMOTE\"," + + " \"localRepo\": \"foo\"," + + " \"localPath\": \"/mirrors/foo\"," + + " \"remoteUri\": \"git+ssh://foo.com/foo.git\"," + + " \"credentialId\": \"alice\"" + + '}'), + Change.ofJsonUpsert( + "/mirrors/bar.json", + '{' + + " \"id\": \"bar\"," + + " \"enabled\": true," + + " \"schedule\": \"0 */10 * * * ?\"," + + " \"direction\": \"REMOTE_TO_LOCAL\"," + + " \"localRepo\": \"bar\"," + + " \"remoteUri\": \"git+ssh://bar.com/bar.git/some-path\"," + + " \"credentialId\": \"bob\"" + + '}'), + Change.ofJsonUpsert( + "/mirrors/qux.json", + '{' + + " \"id\": \"qux\"," + + " \"direction\": \"LOCAL_TO_REMOTE\"," + + " \"localRepo\": \"qux\"," + + " \"remoteUri\": \"git+ssh://qux.net/qux.git#develop\"" + + // No credential will be chosen. + '}'), + Change.ofJsonUpsert( + "/mirrors/foo-bar.json", + '{' + + " \"id\": \"foo-bar\"," + + " \"enabled\": false," + // Disabled + " \"direction\": \"LOCAL_TO_REMOTE\"," + + " \"localRepo\": \"foo\"," + + " \"localPath\": \"/mirrors/bar\"," + + " \"remoteUri\": \"git+ssh://bar.com/bar.git\"" + + // credentialId 'bob' will be chosen. + '}')); + metaRepo.commit(Revision.HEAD, 0L, Author.SYSTEM, "", mirrors).join(); + metaRepo.commit(Revision.HEAD, 0L, Author.SYSTEM, "", UPSERT_RAW_CREDENTIALS).join(); + } else { + final CommandExecutor commandExecutor = pmExtension.executor(); + final List mirrors = ImmutableList.of( + new MirrorDto("foo", true, project.name(), DEFAULT_SCHEDULE, "LOCAL_TO_REMOTE", "foo", + "/mirrors/foo", "git+ssh", "foo.com/foo.git", "", "", null, "alice"), + new MirrorDto("bar", true, project.name(), "0 */10 * * * ?", "REMOTE_TO_LOCAL", "bar", + "", "git+ssh", "bar.com/bar.git", "/some-path", "", null, "bob"), + new MirrorDto("qux", true, project.name(), DEFAULT_SCHEDULE, "LOCAL_TO_REMOTE", "qux", + "", "git+ssh", "qux.net/qux.git", "", "develop", null, ""), + new MirrorDto("foo-bar", false, project.name(), DEFAULT_SCHEDULE, "LOCAL_TO_REMOTE", "foo", + "/mirrors/bar", "git+ssh", "bar.com/bar.git", "", "", null, "bob")); + for (MirrorCredential credential : CREDENTIALS) { + final Command command = + metaRepo.createPushCommand(credential, Author.SYSTEM, false).join(); + pmExtension.executor().execute(command).join(); + } + for (MirrorDto mirror : mirrors) { + final Command command = + metaRepo.createPushCommand(mirror, Author.SYSTEM, false).join(); + pmExtension.executor().execute(command).join(); + } + } // When the mentioned repositories (foo and bar) do not exist, - assertThat(metaRepo.mirrors()).isEmpty(); + assertThat(metaRepo.mirrors().join()).isEmpty(); project.repos().create("foo", Author.SYSTEM); project.repos().create("bar", Author.SYSTEM); @@ -187,7 +243,7 @@ void testMirror() { assertThat(qux.direction()).isEqualTo(MirrorDirection.LOCAL_TO_REMOTE); assertThat(foo.schedule().equivalent(cronParser.parse("0 * * * * ?"))).isTrue(); - assertThat(bar.schedule().equivalent(cronParser.parse("*/10 * * * * ?"))).isTrue(); + assertThat(bar.schedule().equivalent(cronParser.parse("0 */10 * * * ?"))).isTrue(); assertThat(qux.schedule().equivalent(cronParser.parse("0 * * * * ?"))).isTrue(); assertThat(foo.localPath()).isEqualTo("/mirrors/foo/"); @@ -202,8 +258,8 @@ void testMirror() { assertThat(bar.remotePath()).isEqualTo("/some-path/"); assertThat(qux.remotePath()).isEqualTo("/"); - assertThat(foo.remoteBranch()).isNull(); - assertThat(bar.remoteBranch()).isNull(); + assertThat(foo.remoteBranch()).isEmpty(); + assertThat(bar.remoteBranch()).isEmpty(); assertThat(qux.remoteBranch()).isEqualTo("develop"); // Ensure the credentials are loaded correctly. @@ -226,18 +282,23 @@ void testMirror() { @Test void testMirrorWithCredentialId() { - metaRepo.commit(Revision.HEAD, 0, Author.SYSTEM, "", - Change.ofJsonUpsert( - PATH_MIRRORS, - "[{" + - // type isn't used from https://github.com/line/centraldogma/pull/836 but - // left for backward compatibility check. - " \"type\": \"single\"," + - " \"direction\": \"LOCAL_TO_REMOTE\"," + - " \"localRepo\": \"qux\"," + - " \"remoteUri\": \"git+ssh://qux.net/qux.git\"," + - " \"credentialId\": \"alice\"" + - "}]"), UPSERT_CREDENTIALS).join(); + final List> changes = + ImmutableList.>builder() + .add(Change.ofJsonUpsert( + "/mirrors/foo.json", + '{' + + " \"id\": \"foo\"," + + // type isn't used from https://github.com/line/centraldogma/pull/836 but + // left for backward compatibility check. + " \"type\": \"single\"," + + " \"direction\": \"LOCAL_TO_REMOTE\"," + + " \"localRepo\": \"qux\"," + + " \"remoteUri\": \"git+ssh://qux.net/qux.git\"," + + " \"credentialId\": \"alice\"" + + '}')) + .addAll(UPSERT_RAW_CREDENTIALS) + .build(); + metaRepo.commit(Revision.HEAD, 0, Author.SYSTEM, "", changes).join(); project.repos().create("qux", Author.SYSTEM); @@ -252,8 +313,8 @@ void testMirrorWithCredentialId() { private List findMirrors() { // Get the mirror list and sort it by localRepo name alphabetically for easier testing. - return metaRepo.mirrors().stream() + return metaRepo.mirrors().join().stream() .sorted(Comparator.comparing(m -> m.localRepo().name())) - .collect(Collectors.toList()); + .collect(toImmutableList()); } } diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServiceTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServiceTest.java index 9bc796d3da..8bf2dce232 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServiceTest.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringServiceTest.java @@ -17,11 +17,16 @@ package com.linecorp.centraldogma.server.internal.mirror; import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.File; import java.net.URI; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; @@ -31,9 +36,12 @@ import com.cronutils.model.CronType; import com.cronutils.model.definition.CronDefinitionBuilder; import com.cronutils.parser.CronParser; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; +import com.linecorp.armeria.common.util.UnmodifiableFuture; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.mirror.Mirror; import com.linecorp.centraldogma.server.mirror.MirrorCredential; @@ -42,6 +50,7 @@ import com.linecorp.centraldogma.server.storage.project.ProjectManager; import com.linecorp.centraldogma.server.storage.repository.MetaRepository; import com.linecorp.centraldogma.server.storage.repository.Repository; +import com.linecorp.centraldogma.server.storage.repository.RepositoryManager; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; @@ -59,16 +68,25 @@ void mirroringTaskShouldNeverBeRejected() { final ProjectManager pm = mock(ProjectManager.class); final Project p = mock(Project.class); final MetaRepository mr = mock(MetaRepository.class); + final RepositoryManager rm = mock(RepositoryManager.class); final Repository r = mock(Repository.class); when(pm.list()).thenReturn(ImmutableMap.of("foo", p)); + when(pm.get(anyString())).thenReturn(p); when(p.name()).thenReturn("foo"); + when(p.repos()).thenReturn(rm); + when(rm.get(anyString())).thenReturn(r); when(p.metaRepo()).thenReturn(mr); + when(mr.find(eq(Revision.HEAD), anyString(), anyMap())) + .thenReturn(UnmodifiableFuture.completedFuture(ImmutableMap.of())); + when(r.find(eq(Revision.HEAD), anyString())) + .thenReturn(UnmodifiableFuture.completedFuture(ImmutableMap.of())); when(r.parent()).thenReturn(p); when(r.name()).thenReturn("bar"); - final Mirror mirror = new AbstractMirror(EVERY_SECOND, MirrorDirection.REMOTE_TO_LOCAL, + final Mirror mirror = new AbstractMirror("my-mirror-1", true, EVERY_SECOND, + MirrorDirection.REMOTE_TO_LOCAL, MirrorCredential.FALLBACK, r, "/", - URI.create("unused://uri"), "/", null, null) { + URI.create("unused://uri"), "/", "", null) { @Override protected void mirrorLocalToRemote(File workDir, int maxNumFiles, long maxNumBytes) {} @@ -81,11 +99,13 @@ protected void mirrorRemoteToLocal(File workDir, CommandExecutor executor, } }; - when(mr.mirrors()).thenReturn(ImmutableSet.of(mirror)); + when(mr.mirrors()).thenReturn(CompletableFuture.completedFuture(ImmutableList.of(mirror))); final DefaultMirroringService service = new DefaultMirroringService( temporaryFolder, pm, new SimpleMeterRegistry(), 1, 1, 1); - service.start(mock(CommandExecutor.class)); + final CommandExecutor executor = mock(CommandExecutor.class); + when(executor.execute(any(Command.class))).thenReturn(UnmodifiableFuture.completedFuture(null)); + service.start(executor); try { // The mirroring task should run more than once. diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorTest.java index 928e9ee195..bb800a1e5e 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorTest.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/GitMirrorTest.java @@ -25,8 +25,6 @@ import java.time.ZonedDateTime; -import javax.annotation.Nullable; - import org.junit.jupiter.api.Test; import com.linecorp.centraldogma.server.mirror.Mirror; @@ -39,15 +37,15 @@ class GitMirrorTest { void testGitMirror() { // Simplest possible form assertMirror("git://a.com/b.git", DefaultGitMirror.class, - "git://a.com/b.git", "/", null); + "git://a.com/b.git", "/", ""); // Non-default port number assertMirror("git://a.com:8022/b.git", DefaultGitMirror.class, - "git://a.com:8022/b.git", "/", null); + "git://a.com:8022/b.git", "/", ""); // Non-default remotePath assertMirror("git+http://a.com/b.git/c", DefaultGitMirror.class, - "git+http://a.com/b.git", "/c/", null); + "git+http://a.com/b.git", "/c/", ""); // Non-default remoteBranch assertMirror("git+https://a.com/b.git#develop", DefaultGitMirror.class, @@ -68,15 +66,15 @@ void testGitMirror() { void testSshGitMirror() { // Simplest possible form assertMirror("git+ssh://a.com/b.git", SshGitMirror.class, - "git+ssh://a.com/b.git", "/", null); + "git+ssh://a.com/b.git", "/", ""); // Non-default port number assertMirror("git+ssh://a.com:8022/b.git", SshGitMirror.class, - "git+ssh://a.com:8022/b.git", "/", null); + "git+ssh://a.com:8022/b.git", "/", ""); // Non-default remotePath assertMirror("git+ssh://a.com/b.git/c", SshGitMirror.class, - "git+ssh://a.com/b.git", "/c/", null); + "git+ssh://a.com/b.git", "/c/", ""); // Non-default remoteBranch assertMirror("git+ssh://a.com/b.git#develop", SshGitMirror.class, @@ -102,7 +100,7 @@ void testUnknownScheme() { @Test void jitter() { final AbstractMirror mirror = assertMirror("git://a.com/b.git", AbstractMirror.class, - "git://a.com/b.git", "/", null); + "git://a.com/b.git", "/", ""); assertThat(mirror.schedule()).isSameAs(EVERY_MINUTE); @@ -128,7 +126,7 @@ void jitter() { private static T assertMirror(String remoteUri, Class mirrorType, String expectedRemoteRepoUri, String expectedRemotePath, - @Nullable String expectedRemoteBranch) { + String expectedRemoteBranch) { final Repository repository = mock(Repository.class); final Project project = mock(Project.class); when(repository.parent()).thenReturn(project); diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringAndCredentialServiceV1Test.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringAndCredentialServiceV1Test.java new file mode 100644 index 0000000000..86063b9092 --- /dev/null +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringAndCredentialServiceV1Test.java @@ -0,0 +1,255 @@ +/* + * Copyright 2023 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.mirror; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.fasterxml.jackson.core.JsonParseException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import com.linecorp.armeria.client.BlockingWebClient; +import com.linecorp.armeria.common.HttpHeaderNames; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.ResponseEntity; +import com.linecorp.centraldogma.client.CentralDogma; +import com.linecorp.centraldogma.internal.api.v1.MirrorDto; +import com.linecorp.centraldogma.internal.api.v1.PushResultDto; +import com.linecorp.centraldogma.server.internal.mirror.credential.AccessTokenMirrorCredential; +import com.linecorp.centraldogma.server.internal.mirror.credential.NoneMirrorCredential; +import com.linecorp.centraldogma.server.internal.mirror.credential.PasswordMirrorCredential; +import com.linecorp.centraldogma.server.internal.mirror.credential.PublicKeyMirrorCredential; +import com.linecorp.centraldogma.server.mirror.MirrorCredential; +import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; + +class MirroringAndCredentialServiceV1Test { + + private static final String FOO_PROJ = "foo-proj"; + private static final String BAR_REPO = "bar-repo"; + + @RegisterExtension + static final CentralDogmaExtension dogma = new CentralDogmaExtension() { + + @Override + protected void scaffold(CentralDogma client) { + client.createProject(FOO_PROJ).join(); + client.createRepository(FOO_PROJ, BAR_REPO).join(); + } + }; + + private final List hostnamePatterns = ImmutableList.of("github.com"); + private BlockingWebClient client; + + @BeforeEach + void setUp() { + client = dogma.blockingHttpClient(); + } + + @Test + void cruTest() throws JsonParseException { + createAndReadCredential(); + updateCredential(); + createAndReadMirror(); + updateMirror(); + } + + private void createAndReadCredential() { + final List> credentials = ImmutableList.of( + ImmutableMap.of("type", "password", "id", "password-credential", + "hostnamePatterns", hostnamePatterns, + "username", "username-0", "password", "password-0"), + ImmutableMap.of("type", "access_token", "id", "access-token-credential", "hostnamePatterns", + hostnamePatterns, "accessToken", "secret-token-abc-1"), + ImmutableMap.of("type", "public_key", "id", "public-key-credential", + "hostnamePatterns", hostnamePatterns, "username", "username-2", + "publicKey", "public-key-2", "privateKey", "private-key-2", + "passphrase", "password-0"), + ImmutableMap.of("type", "none", "id", "non-credential", "hostnamePatterns", hostnamePatterns)); + + for (int i = 0; i < credentials.size(); i++) { + final Map credential = credentials.get(i); + final String credentialId = (String) credential.get("id"); + final ResponseEntity creationResponse = + client.prepare() + .post("/api/v1/projects/{proj}/credentials") + .pathParam("proj", FOO_PROJ) + .header(HttpHeaderNames.AUTHORIZATION, "Bearer anonymous") + .contentJson(credential) + .responseTimeoutMillis(0) + .asJson(PushResultDto.class) + .execute(); + assertThat(creationResponse.status()).isEqualTo(HttpStatus.CREATED); + assertThat(creationResponse.content().revision().major()).isEqualTo(i + 2); + + final ResponseEntity fetchResponse = + client.prepare() + .get("/api/v1/projects/{proj}/credentials/{id}") + .pathParam("proj", FOO_PROJ) + .pathParam("id", credentialId) + .responseTimeoutMillis(0) + .header(HttpHeaderNames.AUTHORIZATION, "Bearer anonymous") + .asJson(MirrorCredential.class) + .execute(); + final MirrorCredential credentialDto = fetchResponse.content(); + assertThat(credentialDto.id()).isEqualTo(credentialId); + assertThat(credentialDto.hostnamePatterns().stream().map(Pattern::pattern)).isEqualTo( + credential.get("hostnamePatterns")); + final String credentialType = (String) credential.get("type"); + if ("password".equals(credentialType)) { + final PasswordMirrorCredential actual = (PasswordMirrorCredential) credentialDto; + assertThat(actual.username()).isEqualTo(credential.get("username")); + assertThat(actual.password()).isEqualTo(credential.get("password")); + } else if ("access_token".equals(credentialType)) { + final AccessTokenMirrorCredential actual = (AccessTokenMirrorCredential) credentialDto; + assertThat(actual.accessToken()).isEqualTo(credential.get("accessToken")); + } else if ("public_key".equals(credentialType)) { + final PublicKeyMirrorCredential actual = (PublicKeyMirrorCredential) credentialDto; + assertThat(actual.username()).isEqualTo(credential.get("username")); + assertThat(actual.publicKey()).isEqualTo(credential.get("publicKey")); + assertThat(actual.rawPrivateKey()).isEqualTo(credential.get("privateKey")); + assertThat(actual.rawPassphrase()).isEqualTo(credential.get("passphrase")); + } else if ("none".equals(credentialType)) { + assertThat(credentialDto).isInstanceOf(NoneMirrorCredential.class); + } else { + throw new AssertionError("Unexpected credential type: " + credential.getClass().getName()); + } + } + } + + private void updateCredential() { + final List hostnamePatterns = ImmutableList.of("gitlab.com"); + final String credentialId = "public-key-credential"; + final Map credential = + ImmutableMap.of("type", "public_key", + "id", credentialId, + "hostnamePatterns", hostnamePatterns, + "username", "updated-username-2", + "publicKey", "updated-public-key-2", + "privateKey", "updated-private-key-2", + "passphrase", "updated-password-0"); + final ResponseEntity creationResponse = + client.prepare() + .put("/api/v1/projects/{proj}/credentials") + .pathParam("proj", FOO_PROJ) + .header(HttpHeaderNames.AUTHORIZATION, "Bearer anonymous") + .contentJson(credential) + .asJson(PushResultDto.class) + .execute(); + assertThat(creationResponse.status()).isEqualTo(HttpStatus.OK); + + final ResponseEntity fetchResponse = + client.prepare() + .get("/api/v1/projects/{proj}/credentials/{id}") + .pathParam("proj", FOO_PROJ) + .pathParam("id", credentialId) + .header(HttpHeaderNames.AUTHORIZATION, "Bearer anonymous") + .asJson(MirrorCredential.class) + .execute(); + final PublicKeyMirrorCredential actual = (PublicKeyMirrorCredential) fetchResponse.content(); + assertThat(actual.id()).isEqualTo((String) credential.get("id")); + assertThat(actual.hostnamePatterns().stream().map(Pattern::pattern)) + .containsExactlyElementsOf(hostnamePatterns); + assertThat(actual.username()).isEqualTo(credential.get("username")); + assertThat(actual.publicKey()).isEqualTo(credential.get("publicKey")); + assertThat(actual.rawPrivateKey()).isEqualTo(credential.get("privateKey")); + assertThat(actual.rawPassphrase()).isEqualTo(credential.get("passphrase")); + } + + private void createAndReadMirror() throws JsonParseException { + for (int i = 0; i < 3; i++) { + final MirrorDto newMirror = newMirror("mirror-" + i); + final ResponseEntity response0 = + client.prepare() + .post("/api/v1/projects/{proj}/mirrors") + .pathParam("proj", FOO_PROJ) + .header(HttpHeaderNames.AUTHORIZATION, "Bearer anonymous") + .contentJson(newMirror) + .asJson(PushResultDto.class) + .execute(); + assertThat(response0.status()).isEqualTo(HttpStatus.CREATED); + final ResponseEntity response1 = + client.prepare() + .get("/api/v1/projects/{proj}/mirrors/{id}") + .pathParam("proj", FOO_PROJ) + .pathParam("id", newMirror.id()) + .header(HttpHeaderNames.AUTHORIZATION, "Bearer anonymous") + .asJson(MirrorDto.class) + .execute(); + final MirrorDto savedMirror = response1.content(); + assertThat(savedMirror).isEqualTo(newMirror); + } + } + + private void updateMirror() { + final MirrorDto mirror = new MirrorDto("mirror-2", + true, + FOO_PROJ, + "5 * * * * ?", + "REMOTE_TO_LOCAL", + BAR_REPO, + "/updated/local-path/", + "git+https", + "github.com/line/centraldogma-updated.git", + "/updated/remote-path/", + "updated-mirror-branch", + ".updated-env", + "access-token-credential"); + final ResponseEntity updateResponse = + client.prepare() + .put("/api/v1/projects/{proj}/mirrors") + .pathParam("proj", FOO_PROJ) + .header(HttpHeaderNames.AUTHORIZATION, "Bearer anonymous") + .contentJson(mirror) + .asJson(PushResultDto.class) + .execute(); + assertThat(updateResponse.status()).isEqualTo(HttpStatus.OK); + final ResponseEntity fetchResponse = + client.prepare() + .get("/api/v1/projects/{proj}/mirrors/{id}") + .pathParam("proj", FOO_PROJ) + .pathParam("id", mirror.id()) + .header(HttpHeaderNames.AUTHORIZATION, "Bearer anonymous") + .asJson(MirrorDto.class) + .execute(); + final MirrorDto savedMirror = fetchResponse.content(); + assertThat(savedMirror).isEqualTo(mirror); + } + + private static MirrorDto newMirror(String id) { + return new MirrorDto(id, + true, + FOO_PROJ, + "5 * * * * ?", + "REMOTE_TO_LOCAL", + BAR_REPO, + "/local-path/" + id + '/', + "git+https", + "github.com/line/centraldogma-authtest.git", + "/remote-path/" + id + '/', + "mirror-branch", + ".my-env0\n.my-env1", + "public-key-credential"); + } +} diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationServiceClusterTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationServiceClusterTest.java new file mode 100644 index 0000000000..4a1bd30a7d --- /dev/null +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationServiceClusterTest.java @@ -0,0 +1,263 @@ +/* + * Copyright 2023 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.mirror; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationService.PATH_LEGACY_CREDENTIALS; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationService.PATH_LEGACY_CREDENTIALS_BACKUP; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationService.PATH_LEGACY_MIRRORS; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationService.PATH_LEGACY_MIRRORS_BACKUP; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.ACCESS_TOKEN_CREDENTIAL; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.PASSWORD_CREDENTIAL; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.PUBLIC_KEY_CREDENTIAL; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.REPO0_MIRROR; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.REPO1_MIRROR; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.REPO2_MIRROR; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.REPO3_MIRROR; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.TEST_PROJ; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.TEST_REPO0; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.TEST_REPO1; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.TEST_REPO2; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.TEST_REPO3; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.assertCredential; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationServiceTest.assertMirrorConfig; +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.getAccessToken; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.IntStream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.fasterxml.jackson.core.JsonParseException; +import com.spotify.futures.CompletableFutures; + +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.armeria.internal.common.util.PortUtil; +import com.linecorp.centraldogma.client.CentralDogmaRepository; +import com.linecorp.centraldogma.client.armeria.ArmeriaCentralDogmaBuilder; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.EntryNotFoundException; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.PathPattern; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.CentralDogma; +import com.linecorp.centraldogma.server.CentralDogmaBuilder; +import com.linecorp.centraldogma.server.ZooKeeperReplicationConfig; +import com.linecorp.centraldogma.server.ZooKeeperServerConfig; +import com.linecorp.centraldogma.server.auth.AuthProviderFactory; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.testing.internal.TemporaryFolderExtension; +import com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil; +import com.linecorp.centraldogma.testing.internal.auth.TestAuthProviderFactory; + +class MirroringMigrationServiceClusterTest { + + @RegisterExtension + static final TemporaryFolderExtension tempDir = new TemporaryFolderExtension(); + private final AuthProviderFactory factory = new TestAuthProviderFactory(); + + @Timeout(value = 3, unit = TimeUnit.MINUTES) + @Test + void shouldMigrateMirrorConfigsWithZooKeeper() throws Exception { + final int numberReplicas = 3; + final int[] ports = unusedTcpPorts(3 * numberReplicas); + final Map serverConfigs = new HashMap<>(); + for (int i = 0; i < numberReplicas; i++) { + final int quorumPort = ports[i * 3]; + final int electionPort = ports[i * 3 + 1]; + serverConfigs.put(i + 1, new ZooKeeperServerConfig("127.0.0.1", quorumPort, electionPort, 0, + null, null)); + } + final List servers = + IntStream.range(0, numberReplicas).mapToObj(i -> { + try { + final File data = tempDir.newFolder().toFile(); + final ZooKeeperReplicationConfig replicationConfig = + new ZooKeeperReplicationConfig(i + 1, serverConfigs); + final int port = ports[i * 3 + 2]; + return new CentralDogmaBuilder(data) + .port(port, SessionProtocol.HTTP) + .mirroringEnabled(false) + .authProviderFactory(factory) + .replication(replicationConfig) + .administrators(TestAuthMessageUtil.USERNAME) + .build(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(toImmutableList()); + final List> futures = servers.stream().map(CentralDogma::start) + .collect(toImmutableList()); + CompletableFutures.allAsList(futures).join(); + + final int serverPort = servers.get(0).activePort().localAddress().getPort(); + final String accessToken = getAccessToken(WebClient.of("http://127.0.0.1:" + serverPort), + TestAuthMessageUtil.USERNAME, + TestAuthMessageUtil.PASSWORD); + final com.linecorp.centraldogma.client.CentralDogma client = + new ArmeriaCentralDogmaBuilder() + .host("127.0.0.1", serverPort) + .accessToken(accessToken) + .build(); + + client.createProject(TEST_PROJ).join(); + client.createRepository(TEST_PROJ, TEST_REPO0).join(); + client.createRepository(TEST_PROJ, TEST_REPO1).join(); + client.createRepository(TEST_PROJ, TEST_REPO2).join(); + client.createRepository(TEST_PROJ, TEST_REPO3).join(); + + final String mirrorsJson = + '[' + REPO0_MIRROR + ',' + REPO1_MIRROR + ',' + REPO2_MIRROR + ',' + REPO3_MIRROR + ']'; + client.push(TEST_PROJ, Project.REPO_META, Revision.HEAD, "Create a new mirrors.json", "", + Markup.PLAINTEXT, Change.ofJsonUpsert(PATH_LEGACY_MIRRORS, mirrorsJson)).join(); + + final String credentialJson = '[' + PUBLIC_KEY_CREDENTIAL + ',' + PASSWORD_CREDENTIAL + ',' + + ACCESS_TOKEN_CREDENTIAL + ']'; + client.push(TEST_PROJ, Project.REPO_META, Revision.HEAD, "Create a new credentials.json", "", + Markup.PLAINTEXT, Change.ofJsonUpsert(PATH_LEGACY_CREDENTIALS, credentialJson)).join(); + // Wait for the replication to complete. + Thread.sleep(5000); + + final List newServers = + servers.stream().parallel().map(server -> { + final int port = server.activePort().localAddress().getPort(); + // Restart the servers with mirroring enabled. + server.stop().join(); + return new CentralDogmaBuilder(server.config().dataDir()) + .port(port, SessionProtocol.HTTP) + .mirroringEnabled(true) + .replication(server.config().replicationConfig()) + .build(); + }) + .collect(toImmutableList()); + + final List> newFutures = newServers.stream().map(CentralDogma::start) + .collect(toImmutableList()); + CompletableFutures.allAsList(newFutures).join(); + // Wait for the mirroring migration to complete. + Thread.sleep(60000); + + // Check if the mirroring migration is correctly completed. + final int newServerPort = newServers.get(0).activePort().localAddress().getPort(); + final com.linecorp.centraldogma.client.CentralDogma newClient = + new ArmeriaCentralDogmaBuilder() + .host("127.0.0.1", newServerPort) + .build(); + final CentralDogmaRepository repo = newClient.forRepo(TEST_PROJ, Project.REPO_META); + + final Map> mirrorEntries = repo.file(PathPattern.of("/mirrors/*.json")).get().join(); + assertThat(mirrorEntries).hasSize(4); + final Map>> mirrors = + mirrorEntries.entrySet().stream() + .collect(toImmutableMap(e -> { + try { + return e.getValue().contentAsJson().get("localRepo").asText(); + } catch (JsonParseException ex) { + throw new RuntimeException(ex); + } + }, Function.identity())); + + assertMirrorConfig(mirrors.get(TEST_REPO0), "mirror-" + TEST_PROJ + '-' + TEST_REPO0 + "-[a-z]+", + REPO0_MIRROR); + assertMirrorConfig(mirrors.get(TEST_REPO1), "mirror-1", REPO1_MIRROR); + // "-1" suffix is added because the mirror ID is duplicated. + assertMirrorConfig(mirrors.get(TEST_REPO2), "mirror-1-1", REPO2_MIRROR); + // "-2" suffix is added because the mirror ID is duplicated. + assertMirrorConfig(mirrors.get(TEST_REPO3), "mirror-1-2", REPO3_MIRROR); + + final Map> credentialEntries = repo.file(PathPattern.of("/credentials/*.json")) + .get() + .join(); + + assertThat(credentialEntries).hasSize(3); + final Map>> credentials = + credentialEntries.entrySet().stream() + .collect(toImmutableMap(e -> { + try { + return e.getValue().contentAsJson().get("type").asText(); + } catch (JsonParseException ex) { + throw new RuntimeException(ex); + } + }, Function.identity())); + + assertCredential(credentials.get("public_key"), "credential-" + TEST_PROJ + "-[a-z]+", + PUBLIC_KEY_CREDENTIAL); + assertCredential(credentials.get("password"), "credential-1", PASSWORD_CREDENTIAL); + // "-1" suffix is added because the credential ID is duplicated. + assertCredential(credentials.get("access_token"), "credential-1-1", ACCESS_TOKEN_CREDENTIAL); + + // Make sure that the legacy files are renamed. + assertThatThrownBy(() -> repo.file(PATH_LEGACY_MIRRORS).get().join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(EntryNotFoundException.class); + assertThatThrownBy(() -> repo.file(PATH_LEGACY_CREDENTIALS).get().join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(EntryNotFoundException.class); + assertThat(repo.file(PATH_LEGACY_MIRRORS_BACKUP).get().join() + .hasContent()).isTrue(); + assertThat(repo.file(PATH_LEGACY_CREDENTIALS_BACKUP).get().join() + .hasContent()).isTrue(); + } + + // Forked from ZooKeeperTestUtil in Armeria. + private static int[] unusedTcpPorts(int numPorts) { + final int[] ports = new int[numPorts]; + for (int i = 0; i < numPorts; i++) { + int mayUnusedTcpPort; + for (;;) { + mayUnusedTcpPort = PortUtil.unusedTcpPort(); + if (i == 0) { + // The first acquired port is always unique. + break; + } + boolean isAcquiredPort = false; + for (int j = 0; j < i; j++) { + isAcquiredPort = ports[j] == mayUnusedTcpPort; + if (isAcquiredPort) { + break; + } + } + + if (isAcquiredPort) { + // Duplicate port. Look up an unused port again. + continue; + } else { + // A newly acquired unique port. + break; + } + } + ports[i] = mayUnusedTcpPort; + } + return ports; + } +} diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationServiceTest.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationServiceTest.java new file mode 100644 index 0000000000..366974bfdb --- /dev/null +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationServiceTest.java @@ -0,0 +1,354 @@ +/* + * Copyright 2023 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.mirror; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationService.MIRROR_MIGRATION_JOB_LOG; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationService.PATH_LEGACY_CREDENTIALS; +import static com.linecorp.centraldogma.server.internal.mirror.MirroringMigrationService.PATH_LEGACY_MIRRORS; +import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson; +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonNode; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.server.metadata.MetadataService; +import com.linecorp.centraldogma.server.mirror.Mirror; +import com.linecorp.centraldogma.server.mirror.MirrorCredential; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.project.ProjectManager; +import com.linecorp.centraldogma.server.storage.repository.Repository; +import com.linecorp.centraldogma.server.storage.repository.RepositoryManager; +import com.linecorp.centraldogma.testing.internal.ProjectManagerExtension; + +class MirroringMigrationServiceTest { + + // The static fields are shared with MirroringMigrationServiceClusterTest. + static final String TEST_PROJ = "fooProj"; + static final String TEST_REPO0 = "repo0"; + static final String TEST_REPO1 = "repo1"; + static final String TEST_REPO2 = "repo2"; + static final String TEST_REPO3 = "repo3"; + + // The real key pair generated using: + // + // ssh-keygen -t rsa -b 768 -N sesame + // + static final String PUBLIC_KEY = + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAYQCmkW9HjZE5q0EM06MUWXYFTNTi" + + "KkfYD/pH2GwJw6yi20Gi0TzjJ6YBLueU48vxkwWmw6sTOEuBxtzefTxs4kQuatev" + + "uXn7tWX9fhSIAEp+zdyQY7InyCqfHFwRwswemCM= trustin@localhost"; + + static final String PRIVATE_KEY = + "-----BEGIN RSA PRIVATE KEY-----\n" + + "Proc-Type: 4,ENCRYPTED\n" + + "DEK-Info: AES-128-CBC,C35856D3C524AA2FD32D878F4409B97E\n" + + '\n' + + "X3HRmqg2bUqfqxkWjHsr4KeN1UyN5QbypGd7Jov/nDSyiIWe4zPJD/3oji0xOK+h\n" + + "Lxq+c8DDu7ItpC6dwe5WexcyIKGF7WqlkqeEhVM3VOkQtbpbdnb7bA8mLja2unMW\n" + + "bFLgQiTF1Y8SlG4Q70N0iY638AeIG/ZUU14LSBFSQDkrtZ+f7bhIhVDDavANMF+B\n" + + "+eiQ4u3W59Cpbm83AfzqotrPXuBusfyBjH7Wfj0XRvOGRjTQT0jXIWWpLqnIy5ms\n" + + "HNGlMoJElUQuPpbQUiFvmqiMj40r9V/Wx/8+GciADOs4FsTvGFKIcouWDhjIWg0b\n" + + "DKFqV/Hw/AjkAafkySxxmk1+EIen4XfkghtlWLwT2Xp4RtJXYiVC9q9483jDv3+Z\n" + + "iTa5rjFuro4WJkDZp6/N6l+/HcbBXL8L6y66xsJwP+6GLuDLpXjGZrneV1ip2dtG\n" + + "BQzvlgCOr9pTAa4Ar7MC3E2C6+qPhOwO4B/f1cigwRaEB92MHz5gJsITU3xVfTjV\n" + + "yf4THKipBDxqnET6F2FMZJFolVzFEXDaCFNC1TjBqS0+A8KaMcO/lXjJxtfvV37l\n" + + "zmB/ey0dZ8WBCazCp9OX3dYgNkVR1yYNlJWOGJS8Cwc=\n" + + "-----END RSA PRIVATE KEY-----"; + + static final String PASSPHRASE = "sesame"; + + // A mirror config without ID + static final String REPO0_MIRROR = + "{\n" + + " \"type\": \"single\",\n" + + " \"enabled\": true,\n" + + " \"schedule\": \"0 * * * * ?\",\n" + + " \"direction\": \"REMOTE_TO_LOCAL\",\n" + + " \"localRepo\": \"" + TEST_REPO0 + "\",\n" + + " \"localPath\": \"/\",\n" + + " \"remoteUri\": \"git+ssh://git.foo.com/foo.git/settings#release\",\n" + + " \"gitignore\": [\n" + + " \"/credential.txt\",\n" + + " \"private_dir\"\n" + + " ]\n" + + '}'; + + // A mirror config with ID + static final String REPO1_MIRROR = + "{\n" + + " \"id\": \"mirror-1\",\n" + + " \"type\": \"single\",\n" + + " \"enabled\": true,\n" + + " \"schedule\": \"0 * * * * ?\",\n" + + " \"direction\": \"REMOTE_TO_LOCAL\",\n" + + " \"localRepo\": \"" + TEST_REPO1 + "\",\n" + + " \"localPath\": \"/\",\n" + + " \"remoteUri\": \"git+ssh://git.bar.com/foo.git/settings#release\",\n" + + " \"credentialId\": \"credential-1\",\n" + + " \"gitignore\": [\n" + + " \"/credential.txt\",\n" + + " \"private_dir\"\n" + + " ]\n" + + '}'; + + // A mirror config with duplicate ID + static final String REPO2_MIRROR = + "{\n" + + " \"id\": \"mirror-1\",\n" + + " \"type\": \"single\",\n" + + " \"enabled\": true,\n" + + " \"schedule\": \"0 * * * * ?\",\n" + + " \"direction\": \"REMOTE_TO_LOCAL\",\n" + + " \"localRepo\": \"" + TEST_REPO2 + "\",\n" + + " \"localPath\": \"/\",\n" + + " \"remoteUri\": \"git+ssh://git.qux.com/foo.git/settings#release\",\n" + + " \"gitignore\": [\n" + + " \"/credential.txt\",\n" + + " \"private_dir\"\n" + + " ]\n" + + '}'; + + // A mirror config with duplicate ID + static final String REPO3_MIRROR = + "{\n" + + " \"id\": \"mirror-1\",\n" + + " \"type\": \"single\",\n" + + " \"enabled\": true,\n" + + " \"schedule\": \"0 * * * * ?\",\n" + + " \"direction\": \"REMOTE_TO_LOCAL\",\n" + + " \"localRepo\": \"" + TEST_REPO3 + "\",\n" + + " \"localPath\": \"/\",\n" + + " \"remoteUri\": \"git+ssh://git.qux.com/foo.git/settings#release\",\n" + + " \"gitignore\": [\n" + + " \"/credential.txt\",\n" + + " \"private_dir\"\n" + + " ]\n" + + '}'; + + // A credential without ID + static final String PUBLIC_KEY_CREDENTIAL = + '{' + + " \"type\": \"public_key\"," + + " \"hostnamePatterns\": [" + + " \"^git\\\\.foo\\\\.com$\"" + + " ]," + + " \"username\": \"trustin\"," + + " \"publicKey\": \"" + Jackson.escapeText(PUBLIC_KEY) + "\"," + + " \"privateKey\": \"" + Jackson.escapeText(PRIVATE_KEY) + "\"," + + " \"passphrase\": \"" + Jackson.escapeText(PASSPHRASE) + '"' + + '}'; + + // A credential with ID + static final String PASSWORD_CREDENTIAL = + '{' + + " \"id\": \"credential-1\"," + + " \"type\": \"password\"," + + " \"hostnamePatterns\": [" + + " \".*.bar\\\\.com$\"" + + " ]," + + " \"username\": \"trustin\"," + + " \"password\": \"sesame\"" + + '}'; + + // A credential with duplicate ID + static final String ACCESS_TOKEN_CREDENTIAL = + '{' + + " \"id\": \"credential-1\"," + + " \"type\": \"access_token\"," + + " \"hostnamePatterns\": [" + + " \"^bar\\\\.com$\"" + + " ]," + + " \"accessToken\": \"sesame\"" + + '}'; + + @RegisterExtension + static ProjectManagerExtension projectManagerExtension = new ProjectManagerExtension() { + @Override + protected boolean runForEachTest() { + return true; + } + + @Override + protected void afterExecutorStarted() { + final ProjectManager projectManager = projectManagerExtension.projectManager(); + final Project project = projectManager.create(TEST_PROJ, Author.SYSTEM); + final RepositoryManager repoManager = project.repos(); + repoManager.create(TEST_REPO0, Author.SYSTEM); + repoManager.create(TEST_REPO1, Author.SYSTEM); + repoManager.create(TEST_REPO2, Author.SYSTEM); + repoManager.create(TEST_REPO3, Author.SYSTEM); + + final MetadataService mds = new MetadataService(projectManager, projectManagerExtension.executor()); + mds.addRepo(Author.SYSTEM, TEST_PROJ, TEST_REPO0).join(); + mds.addRepo(Author.SYSTEM, TEST_PROJ, TEST_REPO1).join(); + mds.addRepo(Author.SYSTEM, TEST_PROJ, TEST_REPO2).join(); + mds.addRepo(Author.SYSTEM, TEST_PROJ, TEST_REPO3).join(); + } + }; + + @Test + void shouldMigrateMirrorsJson() throws Exception { + final ProjectManager projectManager = projectManagerExtension.projectManager(); + final Project project = projectManager.get(TEST_PROJ); + + final String mirrorsJson = + '[' + REPO0_MIRROR + ',' + REPO1_MIRROR + ',' + REPO2_MIRROR + ',' + REPO3_MIRROR + ']'; + project.metaRepo().commit(Revision.HEAD, System.currentTimeMillis(), Author.SYSTEM, + "Create a new mirrors.json", + Change.ofJsonUpsert(PATH_LEGACY_MIRRORS, mirrorsJson)).join(); + final MirroringMigrationService migrationService = new MirroringMigrationService( + projectManager, projectManagerExtension.executor()); + migrationService.migrate(); + + final Map> entries = project.metaRepo() + .find(Revision.HEAD, "/mirrors/*.json") + .join(); + + assertThat(entries).hasSize(4); + final Map>> mirrors = + entries.entrySet().stream() + .collect(toImmutableMap(e -> { + try { + return e.getValue().contentAsJson().get("localRepo").asText(); + } catch (JsonParseException ex) { + throw new RuntimeException(ex); + } + }, Function.identity())); + + assertMirrorConfig(mirrors.get(TEST_REPO0), "mirror-" + TEST_PROJ + '-' + TEST_REPO0 + "-[a-z]+", + REPO0_MIRROR); + assertMirrorConfig(mirrors.get(TEST_REPO1), "mirror-1", REPO1_MIRROR); + // "-1" suffix is added because the mirror ID is duplicated. + assertMirrorConfig(mirrors.get(TEST_REPO2), "mirror-1-1", REPO2_MIRROR); + // "-2" suffix is added because the mirror ID is duplicated. + assertMirrorConfig(mirrors.get(TEST_REPO3), "mirror-1-2", REPO3_MIRROR); + } + + static void assertMirrorConfig(Map.Entry> actualMirrorConfig, String mirrorId, + String expectedMirrorConfig) throws JsonParseException { + assertThat(actualMirrorConfig.getKey()).matches("/mirrors/" + mirrorId + "\\.json"); + final JsonNode mirrorConfig = actualMirrorConfig.getValue().contentAsJson(); + assertThat(mirrorConfig.get("id").asText()).matches(mirrorId); + assertThatJson(mirrorConfig).whenIgnoringPaths("id", "credentialId") + .isEqualTo(expectedMirrorConfig); + } + + @Test + void shouldMigrateCredential() throws Exception { + final ProjectManager projectManager = projectManagerExtension.projectManager(); + final Project project = projectManager.get(TEST_PROJ); + + final String credentialJson = '[' + PUBLIC_KEY_CREDENTIAL + ',' + PASSWORD_CREDENTIAL + ',' + + ACCESS_TOKEN_CREDENTIAL + ']'; + + project.metaRepo().commit(Revision.HEAD, System.currentTimeMillis(), Author.SYSTEM, + "Create a new credentials.json", + Change.ofJsonUpsert(PATH_LEGACY_CREDENTIALS, credentialJson)).join(); + final MirroringMigrationService migrationService = new MirroringMigrationService( + projectManager, projectManagerExtension.executor()); + migrationService.migrate(); + + final Map> entries = project.metaRepo() + .find(Revision.HEAD, "/credentials/*.json") + .join(); + + assertThat(entries).hasSize(3); + final Map>> credentials = + entries.entrySet().stream() + .collect(toImmutableMap(e -> { + try { + return e.getValue().contentAsJson().get("type").asText(); + } catch (JsonParseException ex) { + throw new RuntimeException(ex); + } + }, Function.identity())); + + assertCredential(credentials.get("public_key"), "credential-" + TEST_PROJ + "-[a-z]+", + PUBLIC_KEY_CREDENTIAL); + assertCredential(credentials.get("password"), "credential-1", PASSWORD_CREDENTIAL); + // "-1" suffix is added because the credential ID is duplicated. + assertCredential(credentials.get("access_token"), "credential-1-1", ACCESS_TOKEN_CREDENTIAL); + + // Make sure that the migration log is written. + final Repository dogmaRepo = projectManager.get(InternalProjectInitializer.INTERNAL_PROJECT_DOGMA) + .repos().get(Project.REPO_DOGMA); + final Map> log = dogmaRepo.find(Revision.HEAD, MIRROR_MIGRATION_JOB_LOG).join(); + final JsonNode data = log.get(MIRROR_MIGRATION_JOB_LOG).contentAsJson(); + assertThat(Jackson.readValue(data.get("timestamp").asText(), Instant.class)).isBefore(Instant.now()); + } + + @Test + void shouldUpdateCredentialIdToMirrorConfig() throws Exception { + final ProjectManager projectManager = projectManagerExtension.projectManager(); + final Project project = projectManager.get(TEST_PROJ); + final String mirrorsJson = '[' + REPO0_MIRROR + ',' + REPO1_MIRROR + ',' + REPO2_MIRROR + ']'; + project.metaRepo().commit(Revision.HEAD, System.currentTimeMillis(), Author.SYSTEM, + "Create a new mirrors.json", + Change.ofJsonUpsert(PATH_LEGACY_MIRRORS, mirrorsJson)).join(); + + final String credentialJson = '[' + PUBLIC_KEY_CREDENTIAL + ',' + PASSWORD_CREDENTIAL + ',' + + ACCESS_TOKEN_CREDENTIAL + ']'; + + project.metaRepo().commit(Revision.HEAD, System.currentTimeMillis(), Author.SYSTEM, + "Create a new credentials.json", + Change.ofJsonUpsert(PATH_LEGACY_CREDENTIALS, credentialJson)).join(); + + final MirroringMigrationService migrationService = new MirroringMigrationService( + projectManager, projectManagerExtension.executor()); + migrationService.migrate(); + + final List mirrors = project.metaRepo().mirrors().join(); + assertThat(mirrors).hasSize(3); + for (Mirror mirror : mirrors) { + if ("mirror-1".equals(mirror.id())) { + assertThat(mirror.credential().id()).isEqualTo("credential-1"); + } else if (mirror.id().startsWith("mirror-" + TEST_PROJ + '-' + TEST_REPO0)) { + assertThat(mirror.credential().id()).matches("credential-" + TEST_PROJ + "-[a-z]+"); + } else if (mirror.id().startsWith("mirror-1-")) { + // No matched credential was found. + assertThat(mirror.credential().id()).matches(""); + assertThat(mirror.credential()).isSameAs(MirrorCredential.FALLBACK); + } else { + throw new AssertionError("Unexpected mirror ID: " + mirror.id()); + } + } + } + + static void assertCredential(Map.Entry> actualCredential, String credentialId, + String expectedCredential) throws JsonParseException { + assertThat(actualCredential.getKey()).matches("/credentials/" + credentialId + "\\.json"); + final JsonNode credential = actualCredential.getValue().contentAsJson(); + assertThat(credential.get("id").asText()).matches(credentialId); + assertThatJson(credential).whenIgnoringPaths("id") + .isEqualTo(expectedCredential); + } +} diff --git a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTestUtils.java b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTestUtils.java index 7135de589b..486c055927 100644 --- a/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTestUtils.java +++ b/server-mirror-git/src/test/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTestUtils.java @@ -58,7 +58,7 @@ static T newMirror(String remoteUri, Cron schedule, final MirrorCredential credential = mock(MirrorCredential.class); final Mirror mirror = new GitMirrorProvider().newMirror( - new MirrorContext(schedule, MirrorDirection.LOCAL_TO_REMOTE, + new MirrorContext("mirror-id", true, schedule, MirrorDirection.LOCAL_TO_REMOTE, credential, repository, "/", URI.create(remoteUri), null)); assertThat(mirror).isInstanceOf(mirrorType); @@ -75,7 +75,7 @@ static T newMirror(String remoteUri, Cron schedule, static void assertMirrorNull(String remoteUri) { final MirrorCredential credential = mock(MirrorCredential.class); final Mirror mirror = new GitMirrorProvider().newMirror( - new MirrorContext(EVERY_MINUTE, MirrorDirection.LOCAL_TO_REMOTE, + new MirrorContext("mirror-id", true, EVERY_MINUTE, MirrorDirection.LOCAL_TO_REMOTE, credential, mock(Repository.class), "/", URI.create(remoteUri), null)); assertThat(mirror).isNull(); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java index 16a9956367..0c8e82392e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java @@ -28,7 +28,6 @@ import static com.linecorp.centraldogma.server.auth.AuthProvider.LOGIN_PATH; import static com.linecorp.centraldogma.server.auth.AuthProvider.LOGOUT_API_ROUTES; import static com.linecorp.centraldogma.server.auth.AuthProvider.LOGOUT_PATH; -import static com.linecorp.centraldogma.server.internal.storage.project.ProjectInitializer.initializeInternalProject; import static java.util.Objects.requireNonNull; import java.io.File; @@ -131,8 +130,10 @@ import com.linecorp.centraldogma.server.internal.admin.util.RestfulJsonResponseConverter; import com.linecorp.centraldogma.server.internal.api.AdministrativeService; import com.linecorp.centraldogma.server.internal.api.ContentServiceV1; +import com.linecorp.centraldogma.server.internal.api.CredentialServiceV1; import com.linecorp.centraldogma.server.internal.api.GitHttpService; import com.linecorp.centraldogma.server.internal.api.MetadataApiService; +import com.linecorp.centraldogma.server.internal.api.MirroringServiceV1; import com.linecorp.centraldogma.server.internal.api.ProjectServiceV1; import com.linecorp.centraldogma.server.internal.api.RepositoryServiceV1; import com.linecorp.centraldogma.server.internal.api.TokenService; @@ -144,6 +145,7 @@ import com.linecorp.centraldogma.server.internal.replication.ZooKeeperCommandExecutor; import com.linecorp.centraldogma.server.internal.storage.project.DefaultProjectManager; import com.linecorp.centraldogma.server.internal.storage.project.ProjectApiManager; +import com.linecorp.centraldogma.server.internal.storage.repository.MirrorConfig; import com.linecorp.centraldogma.server.internal.thrift.CentralDogmaExceptionTranslator; import com.linecorp.centraldogma.server.internal.thrift.CentralDogmaServiceImpl; import com.linecorp.centraldogma.server.internal.thrift.CentralDogmaTimeoutScheduler; @@ -152,10 +154,12 @@ import com.linecorp.centraldogma.server.management.ServerStatusManager; import com.linecorp.centraldogma.server.metadata.MetadataService; import com.linecorp.centraldogma.server.metadata.MetadataServiceInjector; +import com.linecorp.centraldogma.server.mirror.MirrorProvider; import com.linecorp.centraldogma.server.plugin.AllReplicasPlugin; import com.linecorp.centraldogma.server.plugin.Plugin; import com.linecorp.centraldogma.server.plugin.PluginInitContext; import com.linecorp.centraldogma.server.plugin.PluginTarget; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import io.micrometer.core.instrument.MeterRegistry; @@ -182,8 +186,23 @@ public class CentralDogma implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(CentralDogma.class); + private static final boolean GIT_MIRROR_ENABLED; + static { Jackson.registerModules(new SimpleModule().addSerializer(CacheStats.class, new CacheStatsSerializer())); + + boolean gitMirrorEnabled = false; + for (MirrorProvider mirrorProvider : MirrorConfig.MIRROR_PROVIDERS) { + if ("com.linecorp.centraldogma.server.internal.mirror.GitMirrorProvider" + .equals(mirrorProvider.getClass().getName())) { + gitMirrorEnabled = true; + break; + } + } + logger.info("Git mirroring: {}", + gitMirrorEnabled ? "enabled" + : "disabled ('centraldogma-server-mirror-git' module is not available)"); + GIT_MIRROR_ENABLED = gitMirrorEnabled; } /** @@ -225,6 +244,8 @@ public static CentralDogma forConfig(File configFile) throws IOException { private SessionManager sessionManager; @Nullable private ServerStatusManager statusManager; + @Nullable + private InternalProjectInitializer projectInitializer; CentralDogma(CentralDogmaConfig cfg, MeterRegistry meterRegistry) { this.cfg = requireNonNull(cfg, "cfg"); @@ -391,14 +412,16 @@ private void doStart() throws Exception { logger.info("Starting the command executor .."); executor = startCommandExecutor(pm, repositoryWorker, purgeWorker, meterRegistry, sessionManager); + // The projectInitializer is set in startCommandExecutor. + assert projectInitializer != null; if (executor.isWritable()) { logger.info("Started the command executor."); - - initializeInternalProject(executor); + projectInitializer.initialize(); } logger.info("Starting the RPC server."); - server = startServer(pm, executor, purgeWorker, meterRegistry, sessionManager); + server = startServer(pm, executor, purgeWorker, meterRegistry, sessionManager, + projectInitializer); logger.info("Started the RPC server at: {}", server.activePorts()); logger.info("Started the Central Dogma successfully."); success = true; @@ -426,32 +449,36 @@ private CommandExecutor startCommandExecutor( if (pluginsForLeaderOnly != null) { logger.info("Starting plugins on the leader replica .."); pluginsForLeaderOnly - .start(cfg, pm, exec, meterRegistry, purgeWorker).handle((unused, cause) -> { - if (cause == null) { - logger.info("Started plugins on the leader replica."); - } else { - logger.error("Failed to start plugins on the leader replica..", cause); - } - return null; - }); + .start(cfg, pm, exec, meterRegistry, purgeWorker, projectInitializer) + .handle((unused, cause) -> { + if (cause == null) { + logger.info("Started plugins on the leader replica."); + } else { + logger.error("Failed to start plugins on the leader replica..", cause); + } + return null; + }); } }; final Consumer onReleaseLeadership = exec -> { if (pluginsForLeaderOnly != null) { logger.info("Stopping plugins on the leader replica .."); - pluginsForLeaderOnly.stop(cfg, pm, exec, meterRegistry, purgeWorker).handle((unused, cause) -> { - if (cause == null) { - logger.info("Stopped plugins on the leader replica."); - } else { - logger.error("Failed to stop plugins on the leader replica.", cause); - } - return null; - }); + pluginsForLeaderOnly.stop(cfg, pm, exec, meterRegistry, purgeWorker, projectInitializer) + .handle((unused, cause) -> { + if (cause == null) { + logger.info("Stopped plugins on the leader replica."); + } else { + logger.error("Failed to stop plugins on the leader replica.", + cause); + } + return null; + }); } }; statusManager = new ServerStatusManager(cfg.dataDir()); + logger.info("Startup mode: {}", statusManager.serverStatus()); final CommandExecutor executor; final ReplicationMethod replicationMethod = cfg.replicationConfig().method(); switch (replicationMethod) { @@ -468,6 +495,7 @@ private CommandExecutor startCommandExecutor( default: throw new Error("unknown replication method: " + replicationMethod); } + projectInitializer = new InternalProjectInitializer(executor); final ServerStatus initialServerStatus = statusManager.serverStatus(); executor.setWritable(initialServerStatus.writable()); @@ -530,7 +558,8 @@ private SessionManager initializeSessionManager() throws Exception { private Server startServer(ProjectManager pm, CommandExecutor executor, ScheduledExecutorService purgeWorker, MeterRegistry meterRegistry, - @Nullable SessionManager sessionManager) { + @Nullable SessionManager sessionManager, + InternalProjectInitializer projectInitializer) { final ServerBuilder sb = Server.builder(); sb.verboseResponses(true); cfg.ports().forEach(sb::port); @@ -606,7 +635,8 @@ private Server startServer(ProjectManager pm, CommandExecutor executor, if (pluginsForAllReplicas != null) { final PluginInitContext pluginInitContext = - new PluginInitContext(config(), pm, executor, meterRegistry, purgeWorker, sb); + new PluginInitContext(config(), pm, executor, meterRegistry, purgeWorker, sb, + projectInitializer); pluginsForAllReplicas.plugins() .forEach(p -> { if (!(p instanceof AllReplicasPlugin)) { @@ -764,6 +794,16 @@ protected HttpResponse doGet(ServiceRequestContext ctx, HttpRequest req) { sb.annotatedService(API_V1_PATH_PREFIX, new RepositoryServiceV1(executor, mds), decorator, v1RequestConverter, jacksonRequestConverterFunction, v1ResponseConverter); + + if (GIT_MIRROR_ENABLED) { + sb.annotatedService(API_V1_PATH_PREFIX, + new MirroringServiceV1(projectApiManager, executor), decorator, + v1RequestConverter, jacksonRequestConverterFunction, v1RequestConverter); + sb.annotatedService(API_V1_PATH_PREFIX, + new CredentialServiceV1(projectApiManager, executor), decorator, + v1RequestConverter, jacksonRequestConverterFunction, v1RequestConverter); + } + sb.annotatedService() .pathPrefix(API_V1_PATH_PREFIX) .defaultServiceNaming(new ServiceNaming() { @@ -933,6 +973,7 @@ private void doStop() { this.pm = null; this.repositoryWorker = null; this.sessionManager = null; + projectInitializer = null; if (meterRegistryToBeClosed != null) { assert meterRegistry instanceof CompositeMeterRegistry; ((CompositeMeterRegistry) meterRegistry).remove(meterRegistryToBeClosed); @@ -1055,7 +1096,8 @@ protected CompletionStage doStart(@Nullable Void unused) throws Exception final CommandExecutor executor = CentralDogma.this.executor; final MeterRegistry meterRegistry = CentralDogma.this.meterRegistry; if (pm != null && executor != null && meterRegistry != null) { - pluginsForAllReplicas.start(cfg, pm, executor, meterRegistry, purgeWorker).join(); + pluginsForAllReplicas.start(cfg, pm, executor, meterRegistry, purgeWorker, + projectInitializer).join(); } } } catch (Exception e) { @@ -1072,7 +1114,8 @@ protected CompletionStage doStop(@Nullable Void unused) throws Exception { final CommandExecutor executor = CentralDogma.this.executor; final MeterRegistry meterRegistry = CentralDogma.this.meterRegistry; if (pm != null && executor != null && meterRegistry != null) { - pluginsForAllReplicas.stop(cfg, pm, executor, meterRegistry, purgeWorker).join(); + pluginsForAllReplicas.stop(cfg, pm, executor, meterRegistry, purgeWorker, + projectInitializer).join(); } } CentralDogma.this.doStop(); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java index b8bf6db0f9..e18529e8b8 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogmaBuilder.java @@ -536,7 +536,16 @@ public CentralDogmaBuilder meterRegistry(MeterRegistry meterRegistry) { } /** - * Sets CORS related configurations. + * Enables CORS with the specified allowed origins. + */ + public CentralDogmaBuilder cors(String... allowedOrigins) { + requireNonNull(allowedOrigins, "allowedOrigins"); + corsConfig = new CorsConfig(ImmutableList.copyOf(allowedOrigins), null); + return this; + } + + /** + * Enables CORS with the specified {@link CorsConfig}. */ public CentralDogmaBuilder cors(CorsConfig corsConfig) { this.corsConfig = requireNonNull(corsConfig, "corsConfig"); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/PluginGroup.java b/server/src/main/java/com/linecorp/centraldogma/server/PluginGroup.java index 4fc9842c88..3da75ca838 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/PluginGroup.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/PluginGroup.java @@ -41,6 +41,7 @@ import com.linecorp.centraldogma.server.plugin.Plugin; import com.linecorp.centraldogma.server.plugin.PluginContext; import com.linecorp.centraldogma.server.plugin.PluginTarget; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import io.micrometer.core.instrument.MeterRegistry; @@ -124,9 +125,10 @@ Optional findFirstPlugin(Class clazz) { */ CompletableFuture start(CentralDogmaConfig config, ProjectManager projectManager, CommandExecutor commandExecutor, MeterRegistry meterRegistry, - ScheduledExecutorService purgeWorker) { + ScheduledExecutorService purgeWorker, + InternalProjectInitializer internalProjectInitializer) { final PluginContext context = new PluginContext(config, projectManager, commandExecutor, meterRegistry, - purgeWorker); + purgeWorker, internalProjectInitializer); return startStop.start(context, context, true); } @@ -135,9 +137,11 @@ CompletableFuture start(CentralDogmaConfig config, ProjectManager projectM */ CompletableFuture stop(CentralDogmaConfig config, ProjectManager projectManager, CommandExecutor commandExecutor, MeterRegistry meterRegistry, - ScheduledExecutorService purgeWorker) { + ScheduledExecutorService purgeWorker, + InternalProjectInitializer internalProjectInitializer) { return startStop.stop( - new PluginContext(config, projectManager, commandExecutor, meterRegistry, purgeWorker)); + new PluginContext(config, projectManager, commandExecutor, meterRegistry, purgeWorker, + internalProjectInitializer)); } private class PluginGroupStartStop extends StartStopSupport { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java index daafce0435..6022d8ef8f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java @@ -15,7 +15,7 @@ */ package com.linecorp.centraldogma.server.command; -import static com.linecorp.centraldogma.server.internal.storage.project.ProjectInitializer.INTERNAL_PROJECT_DOGMA; +import static com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer.INTERNAL_PROJECT_DOGMA; import static java.util.Objects.requireNonNull; import java.util.Map; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/AdministrativeService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/AdministrativeService.java index 8078161f16..7e8e36acf0 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/AdministrativeService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/AdministrativeService.java @@ -18,8 +18,6 @@ import java.util.concurrent.CompletableFuture; -import com.fasterxml.jackson.databind.JsonNode; - import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.server.HttpStatusException; import com.linecorp.armeria.server.annotation.Consumes; @@ -87,8 +85,4 @@ public CompletableFuture updateStatus(UpdateServerStatusRequest st .thenApply(unused -> status()); } } - - private static CompletableFuture rejectStatusPatch(JsonNode patch) { - throw new IllegalArgumentException("Invalid JSON patch: " + patch); - } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java index 09c180f6dd..21b0c4036f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/ContentServiceV1.java @@ -25,7 +25,8 @@ import static com.linecorp.centraldogma.server.internal.api.DtoConverter.convert; import static com.linecorp.centraldogma.server.internal.api.HttpApiUtil.returnOrThrow; import static com.linecorp.centraldogma.server.internal.api.RepositoryServiceV1.increaseCounterIfOldRevisionUsed; -import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.metaRepoFiles; +import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.isMetaFile; +import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.isMirrorFile; import static java.util.Objects.requireNonNull; import java.util.Collection; @@ -59,6 +60,7 @@ import com.linecorp.armeria.server.annotation.RequestConverter; import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.ChangeType; import com.linecorp.centraldogma.common.Entry; import com.linecorp.centraldogma.common.InvalidPushException; import com.linecorp.centraldogma.common.Markup; @@ -76,6 +78,7 @@ import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.internal.admin.auth.AuthUtil; import com.linecorp.centraldogma.server.internal.api.auth.RequiresReadPermission; import com.linecorp.centraldogma.server.internal.api.auth.RequiresWritePermission; import com.linecorp.centraldogma.server.internal.api.converter.ChangesRequestConverter; @@ -84,7 +87,7 @@ import com.linecorp.centraldogma.server.internal.api.converter.QueryRequestConverter; import com.linecorp.centraldogma.server.internal.api.converter.WatchRequestConverter; import com.linecorp.centraldogma.server.internal.api.converter.WatchRequestConverter.WatchRequest; -import com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository; +import com.linecorp.centraldogma.server.metadata.User; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.FindOption; import com.linecorp.centraldogma.server.storage.repository.FindOptions; @@ -188,12 +191,14 @@ private static String normalizePath(String path) { @Post("/projects/{projectName}/repos/{repoName}/contents") @RequiresWritePermission public CompletableFuture push( + ServiceRequestContext ctx, @Param @Default("-1") String revision, Repository repository, Author author, CommitMessageDto commitMessage, @RequestConverter(ChangesRequestConverter.class) Iterable> changes) { - checkPush(repository.name(), changes); + final User user = AuthUtil.currentUser(ctx); + checkPush(repository.name(), changes, user.isAdmin()); meterRegistry.counter("commits.push", "project", repository.parent().name(), "repository", repository.name()) @@ -438,33 +443,58 @@ public CompletableFuture> mergeFiles( * Checks if the commit is for creating a file and raises a {@link InvalidPushException} if the * given {@code repoName} field is one of {@code meta} and {@code dogma} which are internal repositories. */ - public static void checkPush(String repoName, Iterable> changes) { + public static void checkPush(String repoName, Iterable> changes, boolean isAdmin) { if (Project.REPO_META.equals(repoName)) { final boolean hasChangesOtherThanMetaRepoFiles = - Streams.stream(changes).anyMatch(change -> !metaRepoFiles.contains(change.path())); + Streams.stream(changes).anyMatch(change -> !isMetaFile(change.path())); if (hasChangesOtherThanMetaRepoFiles) { throw new InvalidPushException( "The " + Project.REPO_META + " repository is reserved for internal usage."); } + if (isAdmin) { + // Admin may push the legacy files to test the mirror migration. + } else { + for (Change change : changes) { + // 'mirrors.json' and 'credentials.json' are disallowed to be created or modified. + // 'mirrors/{id}.json' and 'credentials/{id}.json' must be used instead. + final String path = change.path(); + if (change.type() == ChangeType.REMOVE) { + continue; + } + if ("/mirrors.json".equals(path)) { + throw new InvalidPushException( + "'/mirrors.json' file is not allowed to create. " + + "Use '/mirrors/{id}.json' file or " + + "'/api/v1/projects/{projectName}/mirrors' API instead."); + } + if ("/credentials.json".equals(path)) { + throw new InvalidPushException( + "'/credentials.json' file is not allowed to create. " + + "Use '/credentials/{id}.json' file or " + + "'/api/v1/projects/{projectName}/credentials' API instead."); + } + } + } + + // TODO(ikhoon): Disallow creating a mirror with the commit API. Mirroring REST API should be used + // to validate the input. final Optional notAllowedLocalRepo = Streams.stream(changes) - .filter(change -> DefaultMetaRepository.PATH_MIRRORS.equals(change.path())) + .filter(change -> isMirrorFile(change.path())) .filter(change -> change.content() != null) .map(change -> { final Object content = change.content(); if (content instanceof JsonNode) { final JsonNode node = (JsonNode) content; - if (!node.isArray()) { + if (!node.isObject()) { return null; } - for (JsonNode jsonNode : node) { - final JsonNode localRepoNode = jsonNode.get(MIRROR_LOCAL_REPO); - if (localRepoNode != null) { - final String localRepo = localRepoNode.textValue(); - if (Project.isReservedRepoName(localRepo)) { - return localRepo; - } + final JsonNode localRepoNode = node.get(MIRROR_LOCAL_REPO); + if (localRepoNode != null) { + final String localRepo = localRepoNode.textValue(); + if (Project.isReservedRepoName(localRepo)) { + return localRepo; } } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/CredentialServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/CredentialServiceV1.java new file mode 100644 index 0000000000..65e1f3a85d --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/CredentialServiceV1.java @@ -0,0 +1,116 @@ +/* + * Copyright 2023 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.api; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import com.linecorp.armeria.server.annotation.ConsumesJson; +import com.linecorp.armeria.server.annotation.ExceptionHandler; +import com.linecorp.armeria.server.annotation.Get; +import com.linecorp.armeria.server.annotation.Param; +import com.linecorp.armeria.server.annotation.Post; +import com.linecorp.armeria.server.annotation.ProducesJson; +import com.linecorp.armeria.server.annotation.Put; +import com.linecorp.armeria.server.annotation.StatusCode; +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.internal.api.v1.PushResultDto; +import com.linecorp.centraldogma.server.command.CommandExecutor; +import com.linecorp.centraldogma.server.internal.api.auth.RequiresReadPermission; +import com.linecorp.centraldogma.server.internal.api.auth.RequiresWritePermission; +import com.linecorp.centraldogma.server.internal.storage.project.ProjectApiManager; +import com.linecorp.centraldogma.server.mirror.MirrorCredential; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.MetaRepository; + +/** + * Annotated service object for managing credential service. + */ +@ProducesJson +@ExceptionHandler(HttpApiExceptionHandler.class) +public class CredentialServiceV1 extends AbstractService { + + private final ProjectApiManager projectApiManager; + + public CredentialServiceV1(ProjectApiManager projectApiManager, CommandExecutor executor) { + super(executor); + this.projectApiManager = projectApiManager; + } + + /** + * GET /projects/{projectName}/credentials + * + *

Returns the list of the credentials in the project. + */ + @RequiresReadPermission(repository = Project.REPO_META) + @Get("/projects/{projectName}/credentials") + public CompletableFuture> listCredentials(@Param String projectName) { + return metaRepo(projectName).credentials(); + } + + /** + * GET /projects/{projectName}/credentials/{id} + * + *

Returns the credential for the ID in the project. + */ + @RequiresReadPermission(repository = Project.REPO_META) + @Get("/projects/{projectName}/credentials/{id}") + public CompletableFuture getCredentialById(@Param String projectName, @Param String id) { + return metaRepo(projectName).credential(id); + } + + /** + * POST /projects/{projectName}/credentials + * + *

Creates a new credential. + */ + @RequiresWritePermission(repository = Project.REPO_META) + @Post("/projects/{projectName}/credentials") + @ConsumesJson + @StatusCode(201) + public CompletableFuture createCredential(@Param String projectName, + MirrorCredential credential, Author author) { + return createOrUpdate(projectName, credential, author, false); + } + + /** + * PUT /projects/{projectName}/credentials + * + *

Update the existing credential. + */ + @RequiresWritePermission(repository = Project.REPO_META) + @Put("/projects/{projectName}/credentials") + @ConsumesJson + public CompletableFuture updateCredential(@Param String projectName, + MirrorCredential credential, Author author) { + return createOrUpdate(projectName, credential, author, true); + } + + private CompletableFuture createOrUpdate(String projectName, + MirrorCredential credential, + Author author, boolean update) { + return metaRepo(projectName).createPushCommand(credential, author, update).thenCompose(command -> { + return executor().execute(command).thenApply(result -> { + return new PushResultDto(result.revision(), command.timestamp()); + }); + }); + } + + private MetaRepository metaRepo(String projectName) { + return projectApiManager.getProject(projectName).metaRepo(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java new file mode 100644 index 0000000000..599fca462e --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MirroringServiceV1.java @@ -0,0 +1,147 @@ +/* + * Copyright 2023 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.api; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +import java.net.URI; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import com.linecorp.armeria.server.annotation.ConsumesJson; +import com.linecorp.armeria.server.annotation.ExceptionHandler; +import com.linecorp.armeria.server.annotation.Get; +import com.linecorp.armeria.server.annotation.Param; +import com.linecorp.armeria.server.annotation.Post; +import com.linecorp.armeria.server.annotation.ProducesJson; +import com.linecorp.armeria.server.annotation.Put; +import com.linecorp.armeria.server.annotation.StatusCode; +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.internal.api.v1.MirrorDto; +import com.linecorp.centraldogma.internal.api.v1.PushResultDto; +import com.linecorp.centraldogma.server.command.CommandExecutor; +import com.linecorp.centraldogma.server.internal.api.auth.RequiresReadPermission; +import com.linecorp.centraldogma.server.internal.api.auth.RequiresWritePermission; +import com.linecorp.centraldogma.server.internal.storage.project.ProjectApiManager; +import com.linecorp.centraldogma.server.mirror.Mirror; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.MetaRepository; + +/** + * Annotated service object for managing mirroring service. + */ +@ProducesJson +@ExceptionHandler(HttpApiExceptionHandler.class) +public class MirroringServiceV1 extends AbstractService { + + // TODO(ikhoon): + // - Write documentation for the REST API specification + // - Add Java APIs to the CentralDogma client + + private final ProjectApiManager projectApiManager; + + public MirroringServiceV1(ProjectApiManager projectApiManager, CommandExecutor executor) { + super(executor); + this.projectApiManager = projectApiManager; + } + + /** + * GET /projects/{projectName}/mirrors + * + *

Returns the list of the mirrors in the project. + */ + @RequiresReadPermission(repository = Project.REPO_META) + @Get("/projects/{projectName}/mirrors") + public CompletableFuture> listMirrors(@Param String projectName) { + return metaRepo(projectName).mirrors(true).thenApply(mirrors -> { + return mirrors.stream() + .map(mirror -> convertToMirrorDto(projectName, mirror)) + .collect(toImmutableList()); + }); + } + + /** + * GET /projects/{projectName}/mirrors/{id} + * + *

Returns the mirror of the ID in the project mirror list. + */ + @RequiresReadPermission(repository = Project.REPO_META) + @Get("/projects/{projectName}/mirrors/{id}") + public CompletableFuture getMirror(@Param String projectName, @Param String id) { + + return metaRepo(projectName).mirror(id).thenApply(mirror -> { + return convertToMirrorDto(projectName, mirror); + }); + } + + /** + * POST /projects/{projectName}/mirrors + * + *

Creates a new mirror. + */ + @RequiresWritePermission(repository = Project.REPO_META) + @Post("/projects/{projectName}/mirrors") + @ConsumesJson + @StatusCode(201) + public CompletableFuture createMirror(@Param String projectName, MirrorDto newMirror, + Author author) { + return createOrUpdate(projectName, newMirror, author, false); + } + + /** + * PUT /projects/{projectName}/mirrors + * + *

Update the exising mirror. + */ + @RequiresWritePermission(repository = Project.REPO_META) + @Put("/projects/{projectName}/mirrors") + @ConsumesJson + public CompletableFuture updateMirror(@Param String projectName, MirrorDto mirror, + Author author) { + return createOrUpdate(projectName, mirror, author, true); + } + + private CompletableFuture createOrUpdate(String projectName, + MirrorDto newMirror, + Author author, boolean update) { + return metaRepo(projectName).createPushCommand(newMirror, author, update).thenCompose(command -> { + return executor().execute(command).thenApply(result -> { + return new PushResultDto(result.revision(), command.timestamp()); + }); + }); + } + + private static MirrorDto convertToMirrorDto(String projectName, Mirror mirror) { + final URI remoteRepoUri = mirror.remoteRepoUri(); + return new MirrorDto(mirror.id(), + mirror.enabled(), projectName, + mirror.schedule().asString(), + mirror.direction().name(), + mirror.localRepo().name(), + mirror.localPath(), + remoteRepoUri.getScheme(), + remoteRepoUri.getHost() + remoteRepoUri.getPath(), + mirror.remotePath(), + mirror.remoteBranch(), + mirror.gitignore(), + mirror.credential().id()); + } + + private MetaRepository metaRepo(String projectName) { + return projectApiManager.getProject(projectName).metaRepo(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresPermissionDecorator.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresPermissionDecorator.java index c752067ca3..c91b14efb2 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresPermissionDecorator.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresPermissionDecorator.java @@ -25,6 +25,10 @@ import java.util.concurrent.CompletionStage; import java.util.function.Function; +import javax.annotation.Nullable; + +import com.google.common.base.Strings; + import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.HttpStatus; @@ -49,10 +53,18 @@ public final class RequiresPermissionDecorator extends SimpleDecoratingHttpService { private final Permission requiredPermission; - - RequiresPermissionDecorator(HttpService delegate, Permission requiredPermission) { + @Nullable + private final String projectName; + @Nullable + private final String repoName; + + RequiresPermissionDecorator(HttpService delegate, Permission requiredPermission, + @Nullable String projectName, + @Nullable String repoName) { super(delegate); this.requiredPermission = requireNonNull(requiredPermission, "requiredPermission"); + this.projectName = projectName; + this.repoName = repoName; } @Override @@ -60,9 +72,15 @@ public HttpResponse serve(ServiceRequestContext ctx, HttpRequest req) throws Exc final MetadataService mds = MetadataServiceInjector.getMetadataService(ctx); final User user = AuthUtil.currentUser(ctx); - final String projectName = ctx.pathParam("projectName"); + String projectName = this.projectName; + if (projectName == null) { + projectName = ctx.pathParam("projectName"); + } checkArgument(!isNullOrEmpty(projectName), "no project name is specified"); - final String repoName = ctx.pathParam("repoName"); + String repoName = this.repoName; + if (repoName == null) { + repoName = ctx.pathParam("repoName"); + } checkArgument(!isNullOrEmpty(repoName), "no repository name is specified"); if (Project.REPO_DOGMA.equals(repoName)) { @@ -129,7 +147,9 @@ public static final class RequiresReadPermissionDecoratorFactory @Override public Function newDecorator(RequiresReadPermission parameter) { - return delegate -> new RequiresPermissionDecorator(delegate, Permission.READ); + return delegate -> new RequiresPermissionDecorator(delegate, Permission.READ, + Strings.emptyToNull(parameter.project()), + Strings.emptyToNull(parameter.repository())); } } @@ -142,7 +162,9 @@ public static final class RequiresWritePermissionDecoratorFactory @Override public Function newDecorator(RequiresWritePermission parameter) { - return delegate -> new RequiresPermissionDecorator(delegate, Permission.WRITE); + return delegate -> new RequiresPermissionDecorator(delegate, Permission.WRITE, + Strings.emptyToNull(parameter.project()), + Strings.emptyToNull(parameter.repository())); } } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresReadPermission.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresReadPermission.java index 0e55130ce2..784cab0341 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresReadPermission.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresReadPermission.java @@ -24,6 +24,8 @@ import com.linecorp.armeria.server.annotation.DecoratorFactory; import com.linecorp.centraldogma.server.internal.api.auth.RequiresPermissionDecorator.RequiresReadPermissionDecoratorFactory; import com.linecorp.centraldogma.server.metadata.Permission; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.Repository; /** * A {@link Decorator} to allow a request from a user who has a read {@link Permission}. @@ -36,4 +38,16 @@ * A special parameter in order to specify the order of a {@link Decorator}. */ int order() default 0; + + /** + * The name of the {@link Project} to check the permission. + * If not specified, the project name will be extracted from the {@code projectName} path variable. + */ + String project() default ""; + + /** + * The name of the {@link Repository} to check the permission. + * If not specified, the repository name will be extracted from the {@code repoName} path variable. + */ + String repository() default ""; } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresWritePermission.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresWritePermission.java index e1b224a102..4fafbc3501 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresWritePermission.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresWritePermission.java @@ -24,6 +24,8 @@ import com.linecorp.armeria.server.annotation.DecoratorFactory; import com.linecorp.centraldogma.server.internal.api.auth.RequiresPermissionDecorator.RequiresWritePermissionDecoratorFactory; import com.linecorp.centraldogma.server.metadata.Permission; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.Repository; /** * A {@link Decorator} to allow a request from a user who has a write {@link Permission}. @@ -36,4 +38,16 @@ * A special parameter in order to specify the order of a {@link Decorator}. */ int order() default 0; + + /** + * The name of the {@link Project} to check the permission. + * If not specified, the project name will be extracted from the {@code projectName} path variable. + */ + String project() default ""; + + /** + * The name of the {@link Repository} to check the permission. + * If not specified, the repository name will be extracted from the {@code repoName} path variable. + */ + String repository() default ""; } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java index bf00c7a659..6901486376 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java @@ -49,6 +49,8 @@ public abstract class AbstractMirror implements Mirror { protected static final Author MIRROR_AUTHOR = new Author("Mirror", "mirror@localhost.localdomain"); + private final String id; + private final boolean enabled; private final Cron schedule; private final MirrorDirection direction; private final MirrorCredential credential; @@ -56,18 +58,18 @@ public abstract class AbstractMirror implements Mirror { private final String localPath; private final URI remoteRepoUri; private final String remotePath; - @Nullable private final String remoteBranch; @Nullable private final String gitignore; private final ExecutionTime executionTime; private final long jitterMillis; - protected AbstractMirror(Cron schedule, MirrorDirection direction, MirrorCredential credential, - Repository localRepo, String localPath, - URI remoteRepoUri, String remotePath, @Nullable String remoteBranch, + protected AbstractMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction, + MirrorCredential credential, Repository localRepo, String localPath, + URI remoteRepoUri, String remotePath, String remoteBranch, @Nullable String gitignore) { - + this.id = requireNonNull(id, "id"); + this.enabled = enabled; this.schedule = requireNonNull(schedule, "schedule"); this.direction = requireNonNull(direction, "direction"); this.credential = requireNonNull(credential, "credential"); @@ -75,7 +77,7 @@ protected AbstractMirror(Cron schedule, MirrorDirection direction, MirrorCredent this.localPath = normalizePath(requireNonNull(localPath, "localPath")); this.remoteRepoUri = requireNonNull(remoteRepoUri, "remoteRepoUri"); this.remotePath = normalizePath(requireNonNull(remotePath, "remotePath")); - this.remoteBranch = remoteBranch; + this.remoteBranch = requireNonNull(remoteBranch, "remoteBranch"); this.gitignore = gitignore; executionTime = ExecutionTime.forCron(this.schedule); @@ -88,6 +90,11 @@ protected AbstractMirror(Cron schedule, MirrorDirection direction, MirrorCredent (Integer.MAX_VALUE / 60000)); } + @Override + public String id() { + return id; + } + @Override public final Cron schedule() { return schedule; @@ -151,6 +158,11 @@ public final String gitignore() { return gitignore; } + @Override + public final boolean enabled() { + return enabled; + } + @Override public final void mirror(File workDir, CommandExecutor executor, int maxNumFiles, long maxNumBytes) { try { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java index 1a75208268..1d06055707 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java @@ -35,13 +35,13 @@ public final class CentralDogmaMirror extends AbstractMirror { private final String remoteProject; private final String remoteRepo; - public CentralDogmaMirror(Cron schedule, MirrorDirection direction, MirrorCredential credential, - Repository localRepo, String localPath, + public CentralDogmaMirror(String id, boolean enabled, Cron schedule, MirrorDirection direction, + MirrorCredential credential, Repository localRepo, String localPath, URI remoteRepoUri, String remoteProject, String remoteRepo, String remotePath, @Nullable String gitignore) { - // Central Dogma has no notion of 'branch', so we just pass null as a placeholder. - super(schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, null, - gitignore); + // Central Dogma has no notion of 'branch', so we just pass an empty string as a placeholder. + super(id, enabled, schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, + "", gitignore); this.remoteProject = requireNonNull(remoteProject, "remoteProject"); this.remoteRepo = requireNonNull(remoteRepo, "remoteRepo"); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java index fefb7f579c..3214e1edc8 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/DefaultMirroringService.java @@ -22,14 +22,16 @@ import java.io.File; import java.time.Duration; import java.time.ZonedDateTime; -import java.util.Set; +import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import javax.annotation.Nullable; @@ -128,6 +130,15 @@ public synchronized void start(CommandExecutor commandExecutor) { } })); + // Migrate the old mirrors.json to the new format if exists. + try { + new MirroringMigrationService(projectManager, commandExecutor).migrate(); + } catch (Throwable e) { + logger.error("Git mirroring stopped due to an unexpected exception while migrating mirrors.json:", + e); + return; + } + final ListenableScheduledFuture future = scheduler.scheduleWithFixedDelay( this::schedulePendingMirrors, TICK.getSeconds(), TICK.getSeconds(), TimeUnit.SECONDS); @@ -191,9 +202,14 @@ private void schedulePendingMirrors() { projectManager.list() .values() .forEach(project -> { - final Set mirrors; + final List mirrors; try { - mirrors = project.metaRepo().mirrors(); + mirrors = project.metaRepo().mirrors() + .get(5, TimeUnit.SECONDS); + } catch (TimeoutException e) { + logger.warn("Failed to load the mirror list within 5 seconds. project: {}", + project.name(), e); + return; } catch (Exception e) { logger.warn("Failed to load the mirror list from: {}", project.name(), e); return; @@ -217,9 +233,15 @@ public CompletableFuture mirror() { } return CompletableFuture.runAsync( - () -> projectManager.list().values() - .forEach(p -> p.metaRepo().mirrors() - .forEach(m -> run(m, p.name(), false))), + () -> projectManager.list().values().forEach(p -> { + try { + p.metaRepo().mirrors().get(5, TimeUnit.SECONDS) + .forEach(m -> run(m, p.name(), false)); + } catch (InterruptedException | TimeoutException | ExecutionException e) { + throw new IllegalStateException( + "Failed to load mirror list with in 5 seconds. project: " + p.name(), e); + } + }), worker); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationService.java new file mode 100644 index 0000000000..b30b9a4ad7 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirroringMigrationService.java @@ -0,0 +1,464 @@ +/* + * Copyright 2023 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.mirror; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.PATH_CREDENTIALS; +import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.PATH_MIRRORS; +import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.credentialFile; +import static com.linecorp.centraldogma.server.internal.storage.repository.DefaultMetaRepository.mirrorFile; +import static com.linecorp.centraldogma.server.internal.storage.repository.MirrorConfig.DEFAULT_SCHEDULE; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.net.URI; +import java.time.Instant; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.server.command.Command; +import com.linecorp.centraldogma.server.command.CommandExecutor; +import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.internal.replication.ZooKeeperCommandExecutor; +import com.linecorp.centraldogma.server.internal.storage.repository.MirrorConfig; +import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryMetadataException; +import com.linecorp.centraldogma.server.management.ServerStatus; +import com.linecorp.centraldogma.server.mirror.MirrorCredential; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.project.ProjectManager; +import com.linecorp.centraldogma.server.storage.repository.MetaRepository; +import com.linecorp.centraldogma.server.storage.repository.Repository; + +final class MirroringMigrationService { + + private static final Logger logger = LoggerFactory.getLogger(MirroringMigrationService.class); + + @VisibleForTesting + static final String PATH_LEGACY_MIRRORS = "/mirrors.json"; + @VisibleForTesting + static final String PATH_LEGACY_CREDENTIALS = "/credentials.json"; + @VisibleForTesting + static final String PATH_LEGACY_MIRRORS_BACKUP = PATH_LEGACY_MIRRORS + ".bak"; + @VisibleForTesting + static final String PATH_LEGACY_CREDENTIALS_BACKUP = PATH_LEGACY_CREDENTIALS + ".bak"; + @VisibleForTesting + static final String MIRROR_MIGRATION_JOB_LOG = "/mirror-migration-job.json"; + + private final ProjectManager projectManager; + private final CommandExecutor commandExecutor; + + @Nullable + private List shortWords; + + MirroringMigrationService(ProjectManager projectManager, CommandExecutor commandExecutor) { + this.projectManager = projectManager; + this.commandExecutor = commandExecutor; + } + + void migrate() throws Exception { + if (hasMigrationLog()) { + logger.debug("Mirrors and credentials have already been migrated. Skipping auto migration..."); + return; + } + + // Enter read-only mode. + commandExecutor.execute(Command.updateServerStatus(ServerStatus.REPLICATION_ONLY)) + .get(1, TimeUnit.MINUTES); + logger.info("Starting Mirrors and credentials migration ..."); + if (commandExecutor instanceof ZooKeeperCommandExecutor) { + logger.debug("Waiting for 30 seconds to make sure that all cluster have been notified of the " + + "read-only mode ..."); + Thread.sleep(30000); + } + + final Stopwatch stopwatch = Stopwatch.createStarted(); + int numMigratedProjects = 0; + try { + for (Project project : projectManager.list().values()) { + logger.info("Migrating mirrors and credentials in the project: {} ...", project.name()); + boolean processed = false; + final MetaRepository repository = project.metaRepo(); + processed |= migrateCredentials(repository); + // Update the credential IDs in the mirrors.json file. + processed |= migrateMirrors(repository); + if (processed) { + numMigratedProjects++; + logger.info("Mirrors and credentials in the project: {} have been migrated.", + project.name()); + } else { + logger.info("No legacy configurations of mirrors and credentials found in the project: {}.", + project.name()); + } + } + logMigrationJob(numMigratedProjects); + } catch (Exception ex) { + final MirrorMigrationException mirrorException = new MirrorMigrationException( + "Failed to migrate mirrors and credentials. Rollback to the legacy configurations", ex); + try { + rollbackMigration(); + } catch (Exception ex0) { + ex0.addSuppressed(mirrorException); + throw new MirrorMigrationException("Failed to rollback the mirror migration:", ex0); + } + throw mirrorException; + } + + // Exit read-only mode. + commandExecutor.execute(Command.updateServerStatus(ServerStatus.WRITABLE)) + .get(1, TimeUnit.MINUTES); + logger.info("Mirrors and credentials migration has been completed. (took: {} ms.)", + stopwatch.elapsed().toMillis()); + + shortWords = null; + } + + private void logMigrationJob(int numMigratedProjects) throws Exception { + final ImmutableMap data = + ImmutableMap.of("timestamp", Instant.now(), + "projects", numMigratedProjects); + final Change change = Change.ofJsonUpsert(MIRROR_MIGRATION_JOB_LOG, + Jackson.writeValueAsString(data)); + final Command command = + Command.push(Author.SYSTEM, InternalProjectInitializer.INTERNAL_PROJECT_DOGMA, + Project.REPO_DOGMA, Revision.HEAD, + "Migration of mirrors and credentials has been done", "", + Markup.PLAINTEXT, change); + executeCommand(command); + } + + private void removeMigrationJobLog() throws Exception { + if (!hasMigrationLog()) { + // Maybe the migration job was failed before writing the log. + return; + } + final Change change = Change.ofRemoval(MIRROR_MIGRATION_JOB_LOG); + final Command command = + Command.push(Author.SYSTEM, InternalProjectInitializer.INTERNAL_PROJECT_DOGMA, + Project.REPO_DOGMA, Revision.HEAD, + "Remove the migration job log", "", + Markup.PLAINTEXT, change); + executeCommand(command); + } + + private boolean hasMigrationLog() throws Exception { + final Project internalProj = projectManager.get(InternalProjectInitializer.INTERNAL_PROJECT_DOGMA); + final Repository repository = internalProj.repos().get(Project.REPO_DOGMA); + final Map> entries = repository.find(Revision.HEAD, MIRROR_MIGRATION_JOB_LOG).get(); + final Entry entry = entries.get(MIRROR_MIGRATION_JOB_LOG); + return entry != null; + } + + private boolean migrateMirrors(MetaRepository repository) throws Exception { + final ArrayNode mirrors = getLegacyMetaData(repository, PATH_LEGACY_MIRRORS); + if (mirrors == null) { + return false; + } + + final List credentials = repository.credentials() + .get(30, TimeUnit.SECONDS); + + final Set mirrorIds = new HashSet<>(); + for (JsonNode mirror : mirrors) { + if (!mirror.isObject()) { + logger.warn("A mirror config must be an object: {} (project: {})", mirror, + repository.parent().name()); + continue; + } + try { + migrateMirror(repository, (ObjectNode) mirror, mirrorIds, credentials); + } catch (Exception e) { + logger.warn("Failed to migrate a mirror config: {} (project: {})", mirror, + repository.parent().name(), e); + throw e; + } + } + // Back up the old mirrors.json file and don't use it anymore. + rename(repository, PATH_LEGACY_MIRRORS, PATH_LEGACY_MIRRORS_BACKUP, false); + + return true; + } + + private void migrateMirror(MetaRepository repository, ObjectNode mirror, Set mirrorIds, + List credentials) throws Exception { + String id; + final JsonNode idNode = mirror.get("id"); + if (idNode == null) { + // Fill the 'id' field with a random value if not exists. + id = generateIdForMirror(repository.parent().name(), mirror); + } else { + id = idNode.asText(); + } + id = uniquify(id, mirrorIds); + mirror.put("id", id); + + fillCredentialId(repository, mirror, credentials); + if (mirror.get("schedule") == null) { + mirror.put("schedule", DEFAULT_SCHEDULE); + } + mirrorIds.add(id); + + final String jsonFile = mirrorFile(id); + final Command command = + Command.push(Author.SYSTEM, repository.parent().name(), repository.name(), Revision.HEAD, + "Migrate the mirror " + id + " in '" + PATH_LEGACY_MIRRORS + "' into '" + + jsonFile + "'.", "", Markup.PLAINTEXT, Change.ofJsonUpsert(jsonFile, mirror)); + + executeCommand(command); + } + + private void rollbackMigration() throws Exception { + for (Project project : projectManager.list().values()) { + logger.info("Rolling back the migration of mirrors and credentials in the project: {} ...", + project.name()); + final MetaRepository metaRepository = project.metaRepo(); + rollbackMigration(metaRepository, PATH_MIRRORS, PATH_LEGACY_MIRRORS, + PATH_LEGACY_MIRRORS_BACKUP); + rollbackMigration(metaRepository, PATH_CREDENTIALS, PATH_LEGACY_CREDENTIALS, + PATH_LEGACY_CREDENTIALS_BACKUP); + removeMigrationJobLog(); + } + } + + private void rollbackMigration(MetaRepository repository, String targetDirectory, String originalFile, + String backupFile) throws Exception { + // Delete all files in the target directory + final Map> entries = repository.find(Revision.HEAD, targetDirectory + "**") + .get(); + final List> changes = entries.keySet().stream().map(Change::ofRemoval) + .collect(toImmutableList()); + final Command command = + Command.push(Author.SYSTEM, repository.parent().name(), + repository.name(), Revision.HEAD, + "Rollback the migration of " + targetDirectory, "", + Markup.PLAINTEXT, changes); + try { + executeCommand(command); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new MirrorMigrationException("Failed to rollback the migration of " + targetDirectory, e); + } + // Revert the backup file to the original file if exists. + final Entry backup = repository.getOrNull(Revision.HEAD, backupFile).get(); + if (backup != null) { + rename(repository, backupFile, originalFile, true); + } + } + + private CommitResult executeCommand(Command command) + throws InterruptedException, ExecutionException, TimeoutException { + return commandExecutor.execute(Command.forcePush(command)).get(1, TimeUnit.MINUTES); + } + + private static void fillCredentialId(MetaRepository repository, ObjectNode mirror, + List credentials) { + final JsonNode credentialId = mirror.get("credentialId"); + if (credentialId != null) { + return; + } + final JsonNode remoteUri = mirror.get("remoteUri"); + if (remoteUri == null) { + // An invalid mirror config. + return; + } + + final String remoteUriText = remoteUri.asText(); + final MirrorCredential credential = MirrorConfig.findCredential(credentials, URI.create(remoteUriText), + null); + if (credential == MirrorCredential.FALLBACK) { + logger.warn("Failed to find a credential for the mirror: {}, project: {}. " + + "Using the fallback credential.", mirror, repository.parent().name()); + } + mirror.put("credentialId", credential.id()); + } + + /** + * Migrate the legacy {@code credentials.json} file into the {@code /credentials/.json} directory. + * While migrating, the {@code id} field of each credential is filled with a random value if absent. + */ + private boolean migrateCredentials(MetaRepository repository) throws Exception { + final ArrayNode credentials = getLegacyMetaData(repository, PATH_LEGACY_CREDENTIALS); + if (credentials == null) { + return false; + } + + final Set credentialIds = new HashSet<>(); + int index = 0; + for (JsonNode credential : credentials) { + if (!credential.isObject()) { + logger.warn("A credential config at {} must be an object: {} (project: {})", index, + credential.getNodeType(), + repository.parent().name()); + } else { + try { + migrateCredential(repository, (ObjectNode) credential, credentialIds); + } catch (Exception e) { + logger.warn("Failed to migrate the credential config in project {}", + repository.parent().name(), e); + throw e; + } + } + index++; + } + + // Back up the old credentials.json file and don't use it anymore. + rename(repository, PATH_LEGACY_CREDENTIALS, PATH_LEGACY_CREDENTIALS_BACKUP, false); + return true; + } + + private void migrateCredential(MetaRepository repository, ObjectNode credential, Set credentialIds) + throws Exception { + String id; + final JsonNode idNode = credential.get("id"); + final String projectName = repository.parent().name(); + if (idNode == null) { + // Fill the 'id' field with a random value if not exists. + id = generateIdForCredential(projectName); + } else { + id = idNode.asText(); + } + id = uniquify(id, credentialIds); + credential.put("id", id); + credentialIds.add(id); + + final String jsonFile = credentialFile(id); + final Command command = + Command.push(Author.SYSTEM, projectName, repository.name(), Revision.HEAD, + "Migrate the credential '" + id + "' in '" + PATH_LEGACY_CREDENTIALS + + "' into '" + jsonFile + "'.", "", Markup.PLAINTEXT, + Change.ofJsonUpsert(jsonFile, credential)); + executeCommand(command); + } + + @Nullable + private static ArrayNode getLegacyMetaData(MetaRepository repository, String path) + throws InterruptedException, ExecutionException { + final Map> entries = repository.find(Revision.HEAD, path, ImmutableMap.of()) + .get(); + final Entry entry = entries.get(path); + if (entry == null) { + return null; + } + + final JsonNode content = (JsonNode) entry.content(); + if (!content.isArray()) { + throw new RepositoryMetadataException( + path + " must be an array: " + content.getNodeType()); + } + return (ArrayNode) content; + } + + private CommitResult rename(MetaRepository repository, String oldPath, String newPath, boolean rollback) + throws Exception { + final String summary; + if (rollback) { + summary = "Rollback the migration of " + newPath; + } else { + summary = "Back up the legacy " + oldPath + " into " + newPath; + } + final Command command = Command.push(Author.SYSTEM, repository.parent().name(), + repository.name(), Revision.HEAD, + summary, + "", + Markup.PLAINTEXT, + Change.ofRename(oldPath, newPath)); + return executeCommand(command); + } + + /** + * Generates a reproducible ID for the given mirror. + * Pattern: {@code mirror---}. + */ + private String generateIdForMirror(String projectName, ObjectNode mirror) { + final String id = "mirror-" + projectName + '-' + mirror.get("localRepo").asText(); + return id + '-' + getShortWord(id); + } + + private String getShortWord(String id) { + if (shortWords == null) { + shortWords = buildShortWords(); + } + final int index = Math.abs(id.hashCode()) % shortWords.size(); + return shortWords.get(index); + } + + /** + * Generates a reproducible ID for the given credential. + * Pattern: {@code credential--}. + */ + private String generateIdForCredential(String projectName) { + final String id = "credential-" + projectName; + return id + '-' + getShortWord(projectName); + } + + private static String uniquify(String id, Set existingIds) { + int suffix = 1; + String maybeUnique = id; + while (existingIds.contains(maybeUnique)) { + maybeUnique = id + '-' + suffix++; + } + return maybeUnique; + } + + private static List buildShortWords() { + // TODO(ikhoon) Remove 'short_wordlist.txt' if Central Dogma version has been updated enough and + // we can assume that all users have already migrated. + final InputStream is = MirroringMigrationService.class.getResourceAsStream("short_wordlist.txt"); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is))) { + final ImmutableList.Builder words = ImmutableList.builder(); + words.add(reader.readLine()); + return words.build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static class MirrorMigrationException extends RuntimeException { + + private static final long serialVersionUID = -3924318204193024460L; + + MirrorMigrationException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTask.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTask.java index 1342599b05..ecb53c28c7 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTask.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/MirroringTask.java @@ -16,8 +16,6 @@ package com.linecorp.centraldogma.server.internal.mirror; -import static com.google.common.base.MoreObjects.firstNonNull; - import java.io.File; import com.google.common.collect.ImmutableList; @@ -35,7 +33,7 @@ private static Iterable generateTags(Mirror mirror, String projectName) { return ImmutableList.of( Tag.of("project", projectName), Tag.of("direction", mirror.direction().name()), - Tag.of("remoteBranch", firstNonNull(mirror.remoteBranch(), "")), + Tag.of("remoteBranch", mirror.remoteBranch()), Tag.of("remotePath", mirror.remotePath()), Tag.of("localRepo", mirror.localRepo().name()), Tag.of("localPath", mirror.localPath())); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/AbstractMirrorCredential.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/AbstractMirrorCredential.java index 9e7fdf591c..4e11d08cc9 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/AbstractMirrorCredential.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/AbstractMirrorCredential.java @@ -16,49 +16,62 @@ package com.linecorp.centraldogma.server.internal.mirror.credential; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.linecorp.centraldogma.internal.Util.requireNonNullElements; import static java.util.Objects.requireNonNull; import java.net.URI; -import java.util.Optional; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.Nullable; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.linecorp.centraldogma.server.mirror.MirrorCredential; -abstract class AbstractMirrorCredential implements MirrorCredential { +public abstract class AbstractMirrorCredential implements MirrorCredential { - @Nullable private final String id; + private final boolean enabled; + // TODO(ikhoon): Consider changing 'type' to an enum. + private final String type; private final Set hostnamePatterns; private final Set hostnamePatternStrings; - AbstractMirrorCredential(@Nullable String id, @Nullable Iterable hostnamePatterns) { - this.id = id; + AbstractMirrorCredential(String id, @Nullable Boolean enabled, String type, + @Nullable Iterable hostnamePatterns) { + this.id = requireNonNull(id, "id"); + this.enabled = firstNonNull(enabled, true); + // JsonTypeInfo is ignored when serializing collections. + // As a workaround, manually set the type hint to serialize. + this.type = requireNonNull(type, "type"); this.hostnamePatterns = validateHostnamePatterns(hostnamePatterns); hostnamePatternStrings = this.hostnamePatterns.stream().map(Pattern::pattern) .collect(Collectors.toSet()); } private static Set validateHostnamePatterns(@Nullable Iterable hostnamePatterns) { - if (hostnamePatterns == null) { + if (hostnamePatterns == null || Iterables.isEmpty(hostnamePatterns)) { return ImmutableSet.of(); } - return ImmutableSet.copyOf( requireNonNullElements(hostnamePatterns, "hostnamePatterns")); } @Override - public final Optional id() { - return Optional.ofNullable(id); + public final String id() { + return id; + } + + @JsonProperty("type") + public final String type() { + return type; } @Override @@ -66,6 +79,11 @@ public final Set hostnamePatterns() { return hostnamePatterns; } + @Override + public final boolean enabled() { + return enabled; + } + @Override public final boolean matches(URI uri) { requireNonNull(uri, "uri"); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/AccessTokenMirrorCredential.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/AccessTokenMirrorCredential.java index 00e6749c05..fd5d0ebb68 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/AccessTokenMirrorCredential.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/AccessTokenMirrorCredential.java @@ -38,12 +38,14 @@ public final class AccessTokenMirrorCredential extends AbstractMirrorCredential private final String accessToken; @JsonCreator - public AccessTokenMirrorCredential(@JsonProperty("id") @Nullable String id, + public AccessTokenMirrorCredential(@JsonProperty("id") String id, + @JsonProperty("enabled") @Nullable Boolean enabled, @JsonProperty("hostnamePatterns") @Nullable @JsonDeserialize(contentAs = Pattern.class) Iterable hostnamePatterns, @JsonProperty("accessToken") String accessToken) { - super(id, hostnamePatterns); + super(id, enabled, "access_token", hostnamePatterns); + this.accessToken = requireNonEmpty(accessToken, "accessToken"); } @@ -57,6 +59,11 @@ public String accessToken() { } } + @JsonProperty("accessToken") + public String rawAccessToken() { + return accessToken; + } + @Override public int hashCode() { int result = super.hashCode(); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/NoneMirrorCredential.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/NoneMirrorCredential.java index 0c1f65bab8..b195a81b29 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/NoneMirrorCredential.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/NoneMirrorCredential.java @@ -28,11 +28,12 @@ public final class NoneMirrorCredential extends AbstractMirrorCredential { @JsonCreator - public NoneMirrorCredential(@JsonProperty("id") @Nullable String id, + public NoneMirrorCredential(@JsonProperty("id") String id, + @JsonProperty("enabled") @Nullable Boolean enabled, @JsonProperty("hostnamePatterns") @Nullable @JsonDeserialize(contentAs = Pattern.class) Iterable hostnamePatterns) { - super(id, hostnamePatterns); + super(id, enabled, "none", hostnamePatterns); } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/PasswordMirrorCredential.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/PasswordMirrorCredential.java index f370bed5d9..6c3aa92909 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/PasswordMirrorCredential.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/PasswordMirrorCredential.java @@ -40,18 +40,20 @@ public final class PasswordMirrorCredential extends AbstractMirrorCredential { private final String password; @JsonCreator - public PasswordMirrorCredential(@JsonProperty("id") @Nullable String id, + public PasswordMirrorCredential(@JsonProperty("id") String id, + @JsonProperty("enabled") @Nullable Boolean enabled, @JsonProperty("hostnamePatterns") @Nullable @JsonDeserialize(contentAs = Pattern.class) Iterable hostnamePatterns, @JsonProperty("username") String username, @JsonProperty("password") String password) { - super(id, hostnamePatterns); + super(id, enabled, "password", hostnamePatterns); this.username = requireNonEmpty(username, "username"); this.password = requireNonNull(password, "password"); } + @JsonProperty("username") public String username() { return username; } @@ -67,6 +69,11 @@ public String password() { } } + @JsonProperty("password") + public String rawPassword() { + return password; + } + @Override public int hashCode() { int result = super.hashCode(); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/PublicKeyMirrorCredential.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/PublicKeyMirrorCredential.java index 5c152e0a95..ad27bc0c1f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/PublicKeyMirrorCredential.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/credential/PublicKeyMirrorCredential.java @@ -53,7 +53,8 @@ public final class PublicKeyMirrorCredential extends AbstractMirrorCredential { private final String passphrase; @JsonCreator - public PublicKeyMirrorCredential(@JsonProperty("id") @Nullable String id, + public PublicKeyMirrorCredential(@JsonProperty("id") String id, + @JsonProperty("enabled") @Nullable Boolean enabled, @JsonProperty("hostnamePatterns") @Nullable @JsonDeserialize(contentAs = Pattern.class) Iterable hostnamePatterns, @@ -62,7 +63,7 @@ public PublicKeyMirrorCredential(@JsonProperty("id") @Nullable String id, @JsonProperty("privateKey") String privateKey, @JsonProperty("passphrase") @Nullable String passphrase) { - super(id, hostnamePatterns); + super(id, enabled, "public_key", hostnamePatterns); this.username = requireNonEmpty(username, "username"); this.publicKey = requireNonEmpty(publicKey, "publicKey"); @@ -70,10 +71,12 @@ public PublicKeyMirrorCredential(@JsonProperty("id") @Nullable String id, this.passphrase = passphrase; } + @JsonProperty("username") public String username() { return username; } + @JsonProperty("publicKey") public String publicKey() { return publicKey; } @@ -94,6 +97,11 @@ public List privateKey() { return ImmutableList.copyOf(NEWLINE_SPLITTER.splitToList(converted)); } + @JsonProperty("privateKey") + public String rawPrivateKey() { + return privateKey; + } + @Nullable public String passphrase() { try { @@ -106,6 +114,12 @@ public String passphrase() { } } + @JsonProperty("passphrase") + @Nullable + public String rawPassphrase() { + return passphrase; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java index 9379b3a497..e55056b5c1 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java @@ -18,7 +18,7 @@ import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.linecorp.centraldogma.server.internal.storage.project.ProjectInitializer.INTERNAL_PROJECT_DOGMA; +import static com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer.INTERNAL_PROJECT_DOGMA; import static java.util.Objects.requireNonNull; import java.io.BufferedReader; @@ -405,7 +405,8 @@ protected void doStart(@Nullable Runnable onTakeLeadership, leaderSelector.start(); // Start the delegate. - delegate.start(); + // The delegate is StandaloneCommandExecutor, which will be quite fast to start. + delegate.start().get(); // Get the command executor threads ready. final ThreadPoolExecutor executor = new ThreadPoolExecutor( diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/DefaultProject.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/DefaultProject.java index f25fd78192..991d527a35 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/DefaultProject.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/DefaultProject.java @@ -16,8 +16,8 @@ package com.linecorp.centraldogma.server.internal.storage.project; -import static com.linecorp.centraldogma.server.internal.storage.project.ProjectInitializer.INTERNAL_PROJECT_DOGMA; import static com.linecorp.centraldogma.server.metadata.MetadataService.METADATA_JSON; +import static com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer.INTERNAL_PROJECT_DOGMA; import static java.util.Objects.requireNonNull; import java.io.File; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/ProjectApiManager.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/ProjectApiManager.java index ff8836584c..8f969b6023 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/ProjectApiManager.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/ProjectApiManager.java @@ -15,7 +15,7 @@ */ package com.linecorp.centraldogma.server.internal.storage.project; -import static com.linecorp.centraldogma.server.internal.storage.project.ProjectInitializer.INTERNAL_PROJECT_DOGMA; +import static com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer.INTERNAL_PROJECT_DOGMA; import java.time.Instant; import java.util.Collections; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CentralDogmaMirrorProvider.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CentralDogmaMirrorProvider.java index 4b6c8d79ba..b23961a8c1 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CentralDogmaMirrorProvider.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CentralDogmaMirrorProvider.java @@ -17,7 +17,6 @@ package com.linecorp.centraldogma.server.internal.storage.repository; -import static com.linecorp.centraldogma.server.internal.storage.repository.MirrorUtil.split; import static com.linecorp.centraldogma.server.mirror.MirrorSchemes.SCHEME_DOGMA; import static java.util.Objects.requireNonNull; @@ -47,21 +46,22 @@ public Mirror newMirror(MirrorContext context) { if (!SCHEME_DOGMA.equals(scheme)) { return null; } - final String[] components = split(remoteUri, "dogma"); - final URI remoteRepoUri = URI.create(components[0]); - final Matcher matcher = DOGMA_PATH_PATTERN.matcher(remoteRepoUri.getPath()); - if (!matcher.find()) { + final RepositoryUri repositoryUri = RepositoryUri.parse(remoteUri, "dogma"); + final Matcher pathMatcher = DOGMA_PATH_PATTERN.matcher(repositoryUri.uri().getPath()); + if (!pathMatcher.find()) { + // TODO(ikhoon): Should we use the same resource URI format with Git? + // e.g. dogma://[:].dogma//[/] throw new IllegalArgumentException( "cannot determine project name and repository name: " + remoteUri + " (expected: dogma://[:]//.dogma[])"); } - final String remoteProject = matcher.group(1); - final String remoteRepo = matcher.group(2); - - return new CentralDogmaMirror(context.schedule(), context.direction(), context.credential(), - context.localRepo(), context.localPath(), - remoteRepoUri, remoteProject, remoteRepo, components[1], + final String remoteProject = pathMatcher.group(1); + final String remoteRepo = pathMatcher.group(2); + final String remotePath = repositoryUri.path(); + return new CentralDogmaMirror(context.id(), context.enabled(), context.schedule(), context.direction(), + context.credential(), context.localRepo(), context.localPath(), + repositoryUri.uri(), remoteProject, remoteRepo, remotePath, context.gitignore()); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java index 494117d275..f4674b32cf 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/DefaultMetaRepository.java @@ -16,52 +16,65 @@ package com.linecorp.centraldogma.server.internal.storage.repository; -import java.util.Collections; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; + +import java.net.URI; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import javax.annotation.Nullable; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import com.cronutils.model.Cron; +import com.cronutils.model.field.CronField; +import com.cronutils.model.field.CronFieldName; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableMap; +import com.linecorp.armeria.common.util.Exceptions; +import com.linecorp.armeria.common.util.UnmodifiableFuture; +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.EntryNotFoundException; +import com.linecorp.centraldogma.common.Markup; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.internal.api.v1.MirrorDto; +import com.linecorp.centraldogma.server.command.Command; +import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.mirror.Mirror; import com.linecorp.centraldogma.server.mirror.MirrorCredential; -import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.mirror.MirrorDirection; +import com.linecorp.centraldogma.server.mirror.MirrorUtil; import com.linecorp.centraldogma.server.storage.repository.MetaRepository; import com.linecorp.centraldogma.server.storage.repository.Repository; public final class DefaultMetaRepository extends RepositoryWrapper implements MetaRepository { - public static final String PATH_CREDENTIALS = "/credentials.json"; - - public static final String PATH_MIRRORS = "/mirrors.json"; - - public static final Set metaRepoFiles = ImmutableSet.of(PATH_CREDENTIALS, PATH_MIRRORS); + public static final String PATH_CREDENTIALS = "/credentials/"; - private static final String PATH_CREDENTIALS_AND_MIRRORS = PATH_CREDENTIALS + ',' + PATH_MIRRORS; + public static final String PATH_MIRRORS = "/mirrors/"; - private final Lock mirrorLock = new ReentrantLock(); + public static boolean isMetaFile(String path) { + return "/mirrors.json".equals(path) || "/credentials.json".equals(path) || + (path.endsWith(".json") && (path.startsWith(PATH_CREDENTIALS) || path.startsWith(PATH_MIRRORS))); + } - /** - * The revision number of the /credentials.json and /mirrors.json who generated {@link #mirrors}. - */ - private int mirrorRev = -1; + public static boolean isMirrorFile(String path) { + return path.endsWith(".json") && (path.startsWith(PATH_CREDENTIALS) || path.startsWith(PATH_MIRRORS)); + } - /** - * The repositories of the parent {@link Project} at the moment when {@link #mirrors} is generated. - */ - private Set mirrorRepos = Collections.emptySet(); + public static String credentialFile(String credentialId) { + return PATH_CREDENTIALS + credentialId + ".json"; + } - @Nullable - private Set mirrors; + public static String mirrorFile(String mirrorId) { + return PATH_MIRRORS + mirrorId + ".json"; + } public DefaultMetaRepository(Repository repo) { super(repo); @@ -73,90 +86,232 @@ public org.eclipse.jgit.lib.Repository jGitRepository() { } @Override - public Set mirrors() { - mirrorLock.lock(); - try { - final int headRev = normalizeNow(Revision.HEAD).major(); - final Set repos = parent().repos().list().keySet(); - if (headRev > mirrorRev || !mirrorRepos.equals(repos)) { - mirrors = loadMirrors(headRev); - mirrorRev = headRev; - mirrorRepos = repos; + public CompletableFuture> mirrors(boolean includeDisabled) { + if (includeDisabled) { + return allMirrors(); + } + return allMirrors().thenApply(mirrors -> { + return mirrors.stream().filter(Mirror::enabled).collect(toImmutableList()); + }); + } + + @Override + public CompletableFuture mirror(String id) { + final String mirrorFile = mirrorFile(id); + return find(mirrorFile).thenCompose(entries -> { + @SuppressWarnings("unchecked") + final Entry entry = (Entry) entries.get(mirrorFile); + if (entry == null) { + throw new EntryNotFoundException("failed to find credential '" + mirrorFile + "' in " + + parent().name() + '/' + name()); } - return mirrors; - } finally { - mirrorLock.unlock(); - } + final JsonNode mirrorJson = entry.content(); + if (!mirrorJson.isObject()) { + throw newInvalidJsonTypeException(mirrorFile, mirrorJson); + } + final MirrorConfig c; + try { + c = Jackson.treeToValue(mirrorJson, MirrorConfig.class); + } catch (JsonProcessingException e) { + throw new RepositoryMetadataException("failed to load the mirror configuration", e); + } + + final CompletableFuture> credentials; + if (Strings.isNullOrEmpty(c.credentialId())) { + credentials = credentials(); + } else { + credentials = credential(c.credentialId()).thenApply(ImmutableList::of); + } + return credentials.thenApply(credentials0 -> { + final Mirror mirror = c.toMirror(parent(), credentials0); + if (mirror == null) { + throw new EntryNotFoundException("failed to find a mirror config for '" + mirrorFile + + "' in " + parent().name() + '/' + name()); + } + return mirror; + }); + }); } - private Set loadMirrors(int rev) { - // TODO(trustin): Asynchronization - final Map> entries = - find(new Revision(rev), PATH_CREDENTIALS_AND_MIRRORS, Collections.emptyMap()).join(); + private CompletableFuture> allMirrors() { + return find(PATH_MIRRORS + "*.json").thenCompose(entries -> { + if (entries.isEmpty()) { + return UnmodifiableFuture.completedFuture(ImmutableList.of()); + } - if (!entries.containsKey(PATH_MIRRORS)) { - return Collections.emptySet(); - } + return credentials().thenApply(credentials -> { + try { + return parseMirrors(entries, credentials); + } catch (JsonProcessingException e) { + return Exceptions.throwUnsafely(e); + } + }); + }); + } - final JsonNode mirrorsJson = (JsonNode) entries.get(PATH_MIRRORS).content(); - if (!mirrorsJson.isArray()) { - throw new RepositoryMetadataException( - PATH_MIRRORS + " must be an array: " + mirrorsJson.getNodeType()); - } + private List parseMirrors(Map> entries, List credentials) + throws JsonProcessingException { - if (mirrorsJson.size() == 0) { - return Collections.emptySet(); - } + return entries.entrySet().stream().map(entry -> { + final JsonNode mirrorJson = (JsonNode) entry.getValue().content(); + if (!mirrorJson.isObject()) { + throw newInvalidJsonTypeException(entry.getKey(), mirrorJson); + } + final MirrorConfig c; + try { + c = Jackson.treeToValue(mirrorJson, MirrorConfig.class); + } catch (JsonProcessingException e) { + return Exceptions.throwUnsafely(e); + } + return c.toMirror(parent(), credentials); + }) + .filter(Objects::nonNull) + .collect(toImmutableList()); + } - try { - final List credentials = loadCredentials(entries); - final ImmutableSet.Builder mirrors = ImmutableSet.builder(); + @Override + public CompletableFuture> credentials() { + return find(PATH_CREDENTIALS + "*.json").thenApply(entries -> { + if (entries.isEmpty()) { + return ImmutableList.of(); + } + try { + return parseCredentials(entries); + } catch (Exception e) { + throw new RepositoryMetadataException("failed to load the credential configuration", e); + } + }); + } - for (JsonNode m : mirrorsJson) { - final MirrorConfig c = Jackson.treeToValue(m, MirrorConfig.class); - if (c == null) { - throw new RepositoryMetadataException(PATH_MIRRORS + " contains null."); - } - final Mirror mirror = c.toMirror(parent(), credentials); - if (mirror != null) { - mirrors.add(mirror); - } + @Override + public CompletableFuture credential(String credentialId) { + final String credentialFile = credentialFile(credentialId); + return find(credentialFile).thenApply(entries -> { + @SuppressWarnings("unchecked") + final Entry entry = (Entry) entries.get(credentialFile); + if (entry == null) { + throw new EntryNotFoundException("failed to find credential '" + credentialId + "' in " + + parent().name() + '/' + name()); } - return mirrors.build(); - } catch (RepositoryMetadataException e) { - throw e; - } catch (Exception e) { - throw new RepositoryMetadataException("failed to load the mirror configuration", e); - } + try { + return parseCredential(credentialFile, entry); + } catch (Exception e) { + throw new RepositoryMetadataException("failed to load the credential configuration", e); + } + }); } - private static List loadCredentials(Map> entries) throws Exception { - final Entry e = entries.get(PATH_CREDENTIALS); - if (e == null) { - return Collections.emptyList(); - } + private List parseCredentials(Map> entries) + throws JsonProcessingException { + return entries.entrySet().stream() + .map(entry -> { + try { + //noinspection unchecked + return parseCredential(entry.getKey(), (Entry) entry.getValue()); + } catch (JsonProcessingException e) { + return Exceptions.throwUnsafely(e); + } + }) + .collect(toImmutableList()); + } - final JsonNode credentialsJson = (JsonNode) e.content(); - if (!credentialsJson.isArray()) { - throw new RepositoryMetadataException( - PATH_CREDENTIALS + " must be an array: " + credentialsJson.getNodeType()); + private MirrorCredential parseCredential(String credentialFile, Entry entry) + throws JsonProcessingException { + final JsonNode credentialJson = entry.content(); + if (!credentialJson.isObject()) { + throw newInvalidJsonTypeException(credentialFile, credentialJson); } + return Jackson.treeToValue(credentialJson, MirrorCredential.class); + } - if (credentialsJson.size() == 0) { - return Collections.emptyList(); - } + private RepositoryMetadataException newInvalidJsonTypeException(String fileName, JsonNode credentialJson) { + return new RepositoryMetadataException(parent().name() + '/' + name() + fileName + + " must be an object: " + credentialJson.getNodeType()); + } - final ImmutableList.Builder builder = ImmutableList.builder(); - for (JsonNode c : credentialsJson) { - final MirrorCredential credential = Jackson.treeToValue(c, MirrorCredential.class); - if (credential == null) { - throw new RepositoryMetadataException(PATH_CREDENTIALS + " contains null."); + private CompletableFuture>> find(String filePattern) { + return find(Revision.HEAD, filePattern, ImmutableMap.of()); + } + + @Override + public CompletableFuture> createPushCommand(MirrorDto mirrorDto, Author author, + boolean update) { + validateMirror(mirrorDto); + if (update) { + final String summary = "Update the mirror '" + mirrorDto.id() + '\''; + return mirror(mirrorDto.id()).thenApply(mirror -> { + // Perform the update operation only if the mirror exists. + return newCommand(mirrorDto, author, summary); + }); + } else { + String summary = "Create a new mirror from " + mirrorDto.remoteUrl() + + mirrorDto.remotePath() + '#' + mirrorDto.remoteBranch() + " into " + + mirrorDto.localRepo() + mirrorDto.localPath(); + if (MirrorDirection.valueOf(mirrorDto.direction()) == MirrorDirection.REMOTE_TO_LOCAL) { + summary = "[Remote-to-local] " + summary; + } else { + summary = "[Local-to-remote] " + summary; } - builder.add(credential); + return UnmodifiableFuture.completedFuture(newCommand(mirrorDto, author, summary)); } + } + + @Override + public CompletableFuture> createPushCommand(MirrorCredential credential, + Author author, boolean update) { + checkArgument(!credential.id().isEmpty(), "Credential ID should not be empty"); + + if (update) { + return credential(credential.id()).thenApply(c -> { + assert c.id().equals(credential.id()); + final String summary = "Update the mirror credential '" + credential.id() + '\''; + return newCommand(credential, author, summary); + }); + } else { + final String summary = "Create a new mirror credential for " + credential.id(); + return UnmodifiableFuture.completedFuture(newCommand(credential, author, summary)); + } + } + + private Command newCommand(MirrorDto mirrorDto, Author author, String summary) { + final MirrorConfig mirrorConfig = converterToMirrorConfig(mirrorDto); + final JsonNode jsonNode = Jackson.valueToTree(mirrorConfig); + final Change change = Change.ofJsonUpsert(mirrorFile(mirrorConfig.id()), jsonNode); + return Command.push(author, parent().name(), name(), Revision.HEAD, summary, "", Markup.PLAINTEXT, + change); + } + + private Command newCommand(MirrorCredential credential, Author author, String summary) { + final JsonNode jsonNode = Jackson.valueToTree(credential); + final Change change = Change.ofJsonUpsert(credentialFile(credential.id()), jsonNode); + return Command.push(author, parent().name(), name(), Revision.HEAD, summary, "", Markup.PLAINTEXT, + change); + } + + private static void validateMirror(MirrorDto mirror) { + checkArgument(!Strings.isNullOrEmpty(mirror.id()), "Mirror ID is empty"); + final Cron schedule = MirrorConfig.CRON_PARSER.parse(mirror.schedule()); + final CronField secondField = schedule.retrieve(CronFieldName.SECOND); + checkArgument(!secondField.getExpression().asString().contains("*"), + "The second field of the schedule must be specified. (seconds: *, expected: 0-59)"); + } + + private static MirrorConfig converterToMirrorConfig(MirrorDto mirrorDto) { + final String remoteUri = + mirrorDto.remoteScheme() + "://" + mirrorDto.remoteUrl() + + MirrorUtil.normalizePath(mirrorDto.remotePath()) + '#' + mirrorDto.remoteBranch(); - return builder.build(); + return new MirrorConfig( + mirrorDto.id(), + mirrorDto.enabled(), + mirrorDto.schedule(), + MirrorDirection.valueOf(mirrorDto.direction()), + mirrorDto.localRepo(), + mirrorDto.localPath(), + URI.create(remoteUri), + mirrorDto.gitignore(), + mirrorDto.credentialId()); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java index 8bfb436da7..63be75630c 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorConfig.java @@ -22,7 +22,6 @@ import java.net.URI; import java.util.List; -import java.util.Optional; import java.util.ServiceLoader; import javax.annotation.Nullable; @@ -40,6 +39,7 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Streams; @@ -57,18 +57,19 @@ public final class MirrorConfig { private static final Logger logger = LoggerFactory.getLogger(MirrorConfig.class); - private static final String DEFAULT_SCHEDULE = "0 * * * * ?"; // Every minute + public static final String DEFAULT_SCHEDULE = "0 * * * * ?"; // Every minute - private static final CronParser CRON_PARSER = new CronParser( + public static final CronParser CRON_PARSER = new CronParser( CronDefinitionBuilder.instanceDefinitionFor(CronType.QUARTZ)); - private static final List MIRROR_PROVIDERS; + public static final List MIRROR_PROVIDERS; static { MIRROR_PROVIDERS = ImmutableList.copyOf(ServiceLoader.load(MirrorProvider.class)); logger.debug("Available {}s: {}", MirrorProvider.class.getSimpleName(), MIRROR_PROVIDERS); } + private final String id; private final boolean enabled; private final MirrorDirection direction; @Nullable @@ -82,7 +83,8 @@ public final class MirrorConfig { private final Cron schedule; @JsonCreator - public MirrorConfig(@JsonProperty("enabled") @Nullable Boolean enabled, + public MirrorConfig(@JsonProperty("id") String id, + @JsonProperty("enabled") @Nullable Boolean enabled, @JsonProperty("schedule") @Nullable String schedule, @JsonProperty(value = "direction", required = true) MirrorDirection direction, @JsonProperty(value = "localRepo", required = true) String localRepo, @@ -90,6 +92,7 @@ public MirrorConfig(@JsonProperty("enabled") @Nullable Boolean enabled, @JsonProperty(value = "remoteUri", required = true) URI remoteUri, @JsonProperty("gitignore") @Nullable Object gitignore, @JsonProperty("credentialId") @Nullable String credentialId) { + this.id = requireNonNull(id, "id"); this.enabled = firstNonNull(enabled, true); this.schedule = CRON_PARSER.parse(firstNonNull(schedule, DEFAULT_SCHEDULE)); this.direction = requireNonNull(direction, "direction"); @@ -109,17 +112,17 @@ public MirrorConfig(@JsonProperty("enabled") @Nullable Boolean enabled, } else { this.gitignore = null; } - this.credentialId = credentialId; + this.credentialId = Strings.emptyToNull(credentialId); } @Nullable Mirror toMirror(Project parent, Iterable credentials) { - if (!enabled || localRepo == null || !parent.repos().exists(localRepo)) { + if (localRepo == null || !parent.repos().exists(localRepo)) { return null; } final MirrorContext mirrorContext = new MirrorContext( - schedule, direction, findCredential(credentials, remoteUri, credentialId), + id, enabled, schedule, direction, findCredential(credentials, remoteUri, credentialId), parent.repos().get(localRepo), localPath, remoteUri, gitignore); for (MirrorProvider mirrorProvider : MIRROR_PROVIDERS) { final Mirror mirror = mirrorProvider.newMirror(mirrorContext); @@ -131,13 +134,13 @@ schedule, direction, findCredential(credentials, remoteUri, credentialId), throw new IllegalArgumentException("could not find a mirror provider for " + mirrorContext); } - private static MirrorCredential findCredential(Iterable credentials, URI remoteUri, - @Nullable String credentialId) { + public static MirrorCredential findCredential(Iterable credentials, URI remoteUri, + @Nullable String credentialId) { if (credentialId != null) { // Find by credential ID. for (MirrorCredential c : credentials) { - final Optional id = c.id(); - if (id.isPresent() && credentialId.equals(id.get())) { + final String id = c.id(); + if (credentialId.equals(id)) { return c; } } @@ -153,6 +156,11 @@ private static MirrorCredential findCredential(Iterable creden return MirrorCredential.FALLBACK; } + @JsonProperty("id") + public String id() { + return id; + } + @JsonProperty("enabled") public boolean enabled() { return enabled; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorUtil.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryUri.java similarity index 65% rename from server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorUtil.java rename to server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryUri.java index b40e1f98f6..2afbde2b76 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/MirrorUtil.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryUri.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 LINE Corporation + * Copyright 2023 LINE Corporation * * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance @@ -13,23 +13,25 @@ * License for the specific language governing permissions and limitations * under the License. */ + package com.linecorp.centraldogma.server.internal.storage.repository; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.linecorp.centraldogma.server.mirror.MirrorUtil.normalizePath; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URLDecoder; +import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; -/** - * A utility class for creating a mirroring task. - */ -public final class MirrorUtil { +import com.google.common.base.MoreObjects; + +public final class RepositoryUri { /** - * Splits the specified 'remoteUri' into: + * Parses the specified 'remoteUri' into: * - the actual remote repository URI * - the path in the remote repository * - the branch name. @@ -44,7 +46,7 @@ public final class MirrorUtil { * - remotePath: / (default) * - remoteBranch: {@code defaultBranch} */ - public static String[] split(URI remoteUri, String suffix) { + public static RepositoryUri parse(URI remoteUri, String suffix) { final String host = remoteUri.getHost(); if (host == null && !remoteUri.getScheme().endsWith("+file")) { throw new IllegalArgumentException("no host in remoteUri: " + remoteUri); @@ -85,8 +87,57 @@ public static String[] split(URI remoteUri, String suffix) { final String remoteBranch = remoteUri.getFragment(); - return new String[] { newRemoteUri, remotePath, remoteBranch }; + return new RepositoryUri(URI.create(newRemoteUri), remotePath, firstNonNull(remoteBranch, "")); + } + + private final URI uri; + private final String path; + private final String branch; + + private RepositoryUri(URI uri, String path, String branch) { + this.uri = uri; + this.path = path; + this.branch = branch; + } + + public URI uri() { + return uri; + } + + public String path() { + return path; } - private MirrorUtil() {} + public String branch() { + return branch; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RepositoryUri)) { + return false; + } + + final RepositoryUri that = (RepositoryUri) o; + return uri.equals(that.uri) && + path.equals(that.path) && + branch.equals(that.branch); + } + + @Override + public int hashCode() { + return Objects.hash(uri, path, branch); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("uri", uri) + .add("path", path) + .add("branch", branch) + .toString(); + } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/thrift/CentralDogmaServiceImpl.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/thrift/CentralDogmaServiceImpl.java index 0cf08a2d67..2582b276b9 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/thrift/CentralDogmaServiceImpl.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/thrift/CentralDogmaServiceImpl.java @@ -312,7 +312,7 @@ public void push(String projectName, String repositoryName, Revision baseRevisio final List> convertedChanges = convert(changes, Converter::convert); try { - checkPush(repositoryName, convertedChanges); + checkPush(repositoryName, convertedChanges, false); } catch (Exception e) { resultHandler.onError(e); return; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java index 9a3ec372bc..9003c9ff05 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java @@ -20,10 +20,10 @@ import static com.linecorp.centraldogma.internal.jsonpatch.JsonPatchOperation.asJsonArray; import static com.linecorp.centraldogma.internal.jsonpatch.JsonPatchUtil.encodeSegment; import static com.linecorp.centraldogma.server.internal.storage.project.ProjectApiManager.listProjectsWithoutDogma; -import static com.linecorp.centraldogma.server.internal.storage.project.ProjectInitializer.INTERNAL_PROJECT_DOGMA; import static com.linecorp.centraldogma.server.metadata.RepositorySupport.convertWithJackson; import static com.linecorp.centraldogma.server.metadata.Tokens.SECRET_PREFIX; import static com.linecorp.centraldogma.server.metadata.Tokens.validateSecret; +import static com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer.INTERNAL_PROJECT_DOGMA; import static java.util.Objects.requireNonNull; import java.util.Collection; diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java index e27b427389..0c92be44c7 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/Mirror.java @@ -32,6 +32,11 @@ */ public interface Mirror { + /** + * Returns the ID of the mirroring task. + */ + String id(); + /** * Returns the schedule for the mirroring task. */ @@ -77,7 +82,6 @@ public interface Mirror { /** * Returns the name of the branch in the Git repository where is supposed to be mirrored. */ - @Nullable String remoteBranch(); /** @@ -87,6 +91,11 @@ public interface Mirror { @Nullable String gitignore(); + /** + * Returns whether this {@link Mirror} is enabled. + */ + boolean enabled(); + /** * Performs the mirroring task. * diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java index e9489d82c0..e1a0d4eca8 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorContext.java @@ -32,6 +32,8 @@ */ public final class MirrorContext { + private final String id; + private final boolean enabled; private final Cron schedule; private final MirrorDirection direction; private final MirrorCredential credential; @@ -44,8 +46,11 @@ public final class MirrorContext { /** * Creates a new instance. */ - public MirrorContext(Cron schedule, MirrorDirection direction, MirrorCredential credential, - Repository localRepo, String localPath, URI remoteUri, @Nullable String gitignore) { + public MirrorContext(String id, boolean enabled, Cron schedule, MirrorDirection direction, + MirrorCredential credential, Repository localRepo, String localPath, URI remoteUri, + @Nullable String gitignore) { + this.id = requireNonNull(id, "id"); + this.enabled = enabled; this.schedule = requireNonNull(schedule, "schedule"); this.direction = requireNonNull(direction, "direction"); this.credential = requireNonNull(credential, "credential"); @@ -55,6 +60,20 @@ public MirrorContext(Cron schedule, MirrorDirection direction, MirrorCredential this.gitignore = gitignore; } + /** + * Returns the ID of this mirror. + */ + public String id() { + return id; + } + + /** + * Returns whether this mirror is enabled or not. + */ + public boolean enabled() { + return enabled; + } + /** * Returns the cron schedule of this mirror. */ @@ -108,6 +127,8 @@ public String gitignore() { @Override public String toString() { return MoreObjects.toStringHelper(this).omitNullValues() + .add("id", id) + .add("enabled", enabled) .add("schedule", schedule) .add("direction", direction) .add("credential", credential) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorCredential.java b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorCredential.java index a8151f49b3..1b065370b9 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorCredential.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/mirror/MirrorCredential.java @@ -18,10 +18,11 @@ import java.net.URI; import java.util.Collections; -import java.util.Optional; import java.util.Set; import java.util.regex.Pattern; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -34,27 +35,37 @@ /** * The authentication credentials which are required when accessing the Git repositories. */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "type") @JsonSubTypes({ @Type(value = NoneMirrorCredential.class, name = "none"), @Type(value = PasswordMirrorCredential.class, name = "password"), @Type(value = PublicKeyMirrorCredential.class, name = "public_key"), @Type(value = AccessTokenMirrorCredential.class, name = "access_token") }) +@JsonInclude(JsonInclude.Include.NON_NULL) public interface MirrorCredential { - MirrorCredential FALLBACK = new NoneMirrorCredential(null, Collections.singleton(Pattern.compile("^.*$"))); + MirrorCredential FALLBACK = + new NoneMirrorCredential("", true, Collections.singleton(Pattern.compile("^.*$"))); /** * Returns the ID of the credential. */ - Optional id(); + @JsonProperty("id") + String id(); /** * Returns the {@link Pattern}s compiled from the regular expressions that match a host name. */ + @JsonProperty("hostnamePatterns") Set hostnamePatterns(); + /** + * Returns whether this {@link MirrorCredential} is enabled. + */ + @JsonProperty("enabled") + boolean enabled(); + /** * Returns {@code true} if the specified {@code uri} is matched by one of the host name patterns. * diff --git a/server/src/main/java/com/linecorp/centraldogma/server/plugin/PluginContext.java b/server/src/main/java/com/linecorp/centraldogma/server/plugin/PluginContext.java index 9e667ff7f2..87eca18c40 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/plugin/PluginContext.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/plugin/PluginContext.java @@ -22,6 +22,7 @@ import com.linecorp.centraldogma.server.CentralDogmaConfig; import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.command.CommandExecutor; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.project.ProjectManager; @@ -37,6 +38,7 @@ public class PluginContext { private final CommandExecutor commandExecutor; private final MeterRegistry meterRegistry; private final ScheduledExecutorService purgeWorker; + private final InternalProjectInitializer internalProjectInitializer; /** * Creates a new instance. @@ -51,12 +53,15 @@ public PluginContext(CentralDogmaConfig config, ProjectManager projectManager, CommandExecutor commandExecutor, MeterRegistry meterRegistry, - ScheduledExecutorService purgeWorker) { + ScheduledExecutorService purgeWorker, + InternalProjectInitializer internalProjectInitializer) { this.config = requireNonNull(config, "config"); this.projectManager = requireNonNull(projectManager, "projectManager"); this.commandExecutor = requireNonNull(commandExecutor, "commandExecutor"); this.meterRegistry = requireNonNull(meterRegistry, "meterRegistry"); this.purgeWorker = requireNonNull(purgeWorker, "purgeWorker"); + this.internalProjectInitializer = requireNonNull(internalProjectInitializer, + "internalProjectInitializer"); } /** @@ -93,4 +98,11 @@ public MeterRegistry meterRegistry() { public ScheduledExecutorService purgeWorker() { return purgeWorker; } + + /** + * Returns the {@link InternalProjectInitializer}. + */ + public InternalProjectInitializer internalProjectInitializer() { + return internalProjectInitializer; + } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/plugin/PluginInitContext.java b/server/src/main/java/com/linecorp/centraldogma/server/plugin/PluginInitContext.java index 709decd767..faf230ce65 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/plugin/PluginInitContext.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/plugin/PluginInitContext.java @@ -23,6 +23,7 @@ import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.centraldogma.server.CentralDogmaConfig; import com.linecorp.centraldogma.server.command.CommandExecutor; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import io.micrometer.core.instrument.MeterRegistry; @@ -41,8 +42,9 @@ public PluginInitContext(CentralDogmaConfig config, ProjectManager projectManager, CommandExecutor commandExecutor, MeterRegistry meterRegistry, - ScheduledExecutorService purgeWorker, ServerBuilder serverBuilder) { - super(config, projectManager, commandExecutor, meterRegistry, purgeWorker); + ScheduledExecutorService purgeWorker, ServerBuilder serverBuilder, + InternalProjectInitializer projectInitializer) { + super(config, projectManager, commandExecutor, meterRegistry, purgeWorker, projectInitializer); this.serverBuilder = requireNonNull(serverBuilder, "serverBuilder"); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/ProjectInitializer.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/project/InternalProjectInitializer.java similarity index 69% rename from server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/ProjectInitializer.java rename to server/src/main/java/com/linecorp/centraldogma/server/storage/project/InternalProjectInitializer.java index 8ea609fcca..85cec7afc5 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/project/ProjectInitializer.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/project/InternalProjectInitializer.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 LINE Corporation + * Copyright 2024 LINE Corporation * * LINE Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance @@ -14,11 +14,12 @@ * under the License. */ -package com.linecorp.centraldogma.server.internal.storage.project; +package com.linecorp.centraldogma.server.storage.project; import static com.linecorp.centraldogma.server.command.Command.createProject; import static com.linecorp.centraldogma.server.command.Command.createRepository; import static com.linecorp.centraldogma.server.command.Command.push; +import static java.util.Objects.requireNonNull; import java.util.List; @@ -30,28 +31,52 @@ import com.linecorp.centraldogma.common.ChangeConflictException; import com.linecorp.centraldogma.common.Markup; import com.linecorp.centraldogma.common.ProjectExistsException; +import com.linecorp.centraldogma.common.ReadOnlyException; import com.linecorp.centraldogma.common.RepositoryExistsException; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.metadata.MetadataService; import com.linecorp.centraldogma.server.metadata.Tokens; -import com.linecorp.centraldogma.server.storage.project.Project; -public final class ProjectInitializer { +/** + * Initializes the internal project and repositories. + */ +public final class InternalProjectInitializer { public static final String INTERNAL_PROJECT_DOGMA = "dogma"; + private final CommandExecutor executor; + + /** + * Creates a new instance. + */ + public InternalProjectInitializer(CommandExecutor executor) { + this.executor = executor; + } + /** * Creates an internal project and repositories such as a token storage. */ - public static void initializeInternalProject(CommandExecutor executor) { + public void initialize() { final long creationTimeMillis = System.currentTimeMillis(); - initializeInternalProject(executor, creationTimeMillis, INTERNAL_PROJECT_DOGMA); + try { + executor.execute(createProject(creationTimeMillis, Author.SYSTEM, INTERNAL_PROJECT_DOGMA)) + .get(); + } catch (Throwable cause) { + final Throwable peeled = Exceptions.peel(cause); + if (peeled instanceof ReadOnlyException) { + // The executor has stopped right after starting up. + return; + } + if (!(peeled instanceof ProjectExistsException)) { + throw new Error("failed to initialize an internal project: " + INTERNAL_PROJECT_DOGMA, peeled); + } + } // These repositories might be created when creating an internal project, but we try to create them // again here in order to make sure them exist because sometimes their names are changed. - initializeInternalRepos(executor, creationTimeMillis, Project.internalRepos()); + initializeInternalRepos(Project.internalRepos(), creationTimeMillis); try { final Change change = Change.ofJsonPatch(MetadataService.TOKEN_JSON, @@ -63,27 +88,23 @@ public static void initializeInternalProject(CommandExecutor executor) { .get(); } catch (Throwable cause) { final Throwable peeled = Exceptions.peel(cause); - if (!(peeled instanceof ChangeConflictException)) { - throw new Error("failed to initialize the token list file", peeled); + if (peeled instanceof ReadOnlyException || peeled instanceof ChangeConflictException) { + return; } + throw new Error("failed to initialize the token list file", peeled); } } - public static void initializeInternalProject( - CommandExecutor executor, long creationTimeMillis, String projectName) { - try { - executor.execute(createProject(creationTimeMillis, Author.SYSTEM, projectName)) - .get(); - } catch (Throwable cause) { - final Throwable peeled = Exceptions.peel(cause); - if (!(peeled instanceof ProjectExistsException)) { - throw new Error("failed to initialize an internal project: " + projectName, peeled); - } - } + /** + * Creates the specified internal repositories in the internal project. + */ + public void initializeInternalRepos(List internalRepos) { + requireNonNull(internalRepos, "internalRepos"); + final long creationTimeMillis = System.currentTimeMillis(); + initializeInternalRepos(internalRepos, creationTimeMillis); } - public static void initializeInternalRepos( - CommandExecutor executor, long creationTimeMillis, List internalRepos) { + private void initializeInternalRepos(List internalRepos, long creationTimeMillis) { for (final String repo : internalRepos) { try { executor.execute(createRepository(creationTimeMillis, Author.SYSTEM, @@ -91,6 +112,10 @@ public static void initializeInternalRepos( .get(); } catch (Throwable cause) { final Throwable peeled = Exceptions.peel(cause); + if (peeled instanceof ReadOnlyException) { + // The executor has stopped right after starting up. + return; + } if (!(peeled instanceof RepositoryExistsException)) { throw new Error("failed to initialize an internal repository: " + INTERNAL_PROJECT_DOGMA + '/' + repo, peeled); @@ -98,6 +123,4 @@ public static void initializeInternalRepos( } } } - - private ProjectInitializer() {} } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/MetaRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/MetaRepository.java index 3657c6cb7a..1f54a1f41e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/MetaRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/MetaRepository.java @@ -16,9 +16,15 @@ package com.linecorp.centraldogma.server.storage.repository; -import java.util.Set; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.internal.api.v1.MirrorDto; +import com.linecorp.centraldogma.server.command.Command; +import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.mirror.Mirror; +import com.linecorp.centraldogma.server.mirror.MirrorCredential; /** * A Revision-controlled filesystem-like repository which is named {@code "meta"}. @@ -26,7 +32,42 @@ public interface MetaRepository extends Repository { /** - * Returns a set of mirroring tasks. + * Returns active mirroring tasks. */ - Set mirrors(); + default CompletableFuture> mirrors() { + return mirrors(false); + } + + /** + * Returns a set of mirroring tasks. If {@code includeDisabled} is {@code true}, disabled mirroring tasks + * are also included in the returned {@link Mirror}s. + */ + CompletableFuture> mirrors(boolean includeDisabled); + + /** + * Returns a mirroring task of the specified {@code id}. + */ + CompletableFuture mirror(String id); + + /** + * Returns a list of mirroring credentials. + */ + CompletableFuture> credentials(); + + /** + * Returns a mirroring credential of the specified {@code id}. + */ + CompletableFuture credential(String id); + + /** + * Create a push {@link Command} for the {@link MirrorDto}. + */ + CompletableFuture> createPushCommand(MirrorDto mirrorDto, Author author, + boolean update); + + /** + * Create a push {@link Command} for the {@link MirrorCredential}. + */ + CompletableFuture> createPushCommand(MirrorCredential credential, Author author, + boolean update); } diff --git a/server/src/main/resources/com/linecorp/centraldogma/server/internal/mirror/short_wordlist.txt b/server/src/main/resources/com/linecorp/centraldogma/server/internal/mirror/short_wordlist.txt new file mode 100644 index 0000000000..9ac732fe36 --- /dev/null +++ b/server/src/main/resources/com/linecorp/centraldogma/server/internal/mirror/short_wordlist.txt @@ -0,0 +1,1296 @@ +aardvark +abandoned +abbreviate +abdomen +abhorrence +abiding +abnormal +abrasion +absorbing +abundant +abyss +academy +accountant +acetone +achiness +acid +acoustics +acquire +acrobat +actress +acuteness +aerosol +aesthetic +affidavit +afloat +afraid +aftershave +again +agency +aggressor +aghast +agitate +agnostic +agonizing +agreeing +aidless +aimlessly +ajar +alarmclock +albatross +alchemy +alfalfa +algae +aliens +alkaline +almanac +alongside +alphabet +already +also +altitude +aluminum +always +amazingly +ambulance +amendment +amiable +ammunition +amnesty +amoeba +amplifier +amuser +anagram +anchor +android +anesthesia +angelfish +animal +anklet +announcer +anonymous +answer +antelope +anxiety +anyplace +aorta +apartment +apnea +apostrophe +apple +apricot +aquamarine +arachnid +arbitrate +ardently +arena +argument +aristocrat +armchair +aromatic +arrowhead +arsonist +artichoke +asbestos +ascend +aseptic +ashamed +asinine +asleep +asocial +asparagus +astronaut +asymmetric +atlas +atmosphere +atom +atrocious +attic +atypical +auctioneer +auditorium +augmented +auspicious +automobile +auxiliary +avalanche +avenue +aviator +avocado +awareness +awhile +awkward +awning +awoke +axially +azalea +babbling +backpack +badass +bagpipe +bakery +balancing +bamboo +banana +barracuda +basket +bathrobe +bazooka +blade +blender +blimp +blouse +blurred +boatyard +bobcat +body +bogusness +bohemian +boiler +bonnet +boots +borough +bossiness +bottle +bouquet +boxlike +breath +briefcase +broom +brushes +bubblegum +buckle +buddhist +buffalo +bullfrog +bunny +busboy +buzzard +cabin +cactus +cadillac +cafeteria +cage +cahoots +cajoling +cakewalk +calculator +camera +canister +capsule +carrot +cashew +cathedral +caucasian +caviar +ceasefire +cedar +celery +cement +census +ceramics +cesspool +chalkboard +cheesecake +chimney +chlorine +chopsticks +chrome +chute +cilantro +cinnamon +circle +cityscape +civilian +clay +clergyman +clipboard +clock +clubhouse +coathanger +cobweb +coconut +codeword +coexistent +coffeecake +cognitive +cohabitate +collarbone +computer +confetti +copier +cornea +cosmetics +cotton +couch +coverless +coyote +coziness +crawfish +crewmember +crib +croissant +crumble +crystal +cubical +cucumber +cuddly +cufflink +cuisine +culprit +cup +curry +cushion +cuticle +cybernetic +cyclist +cylinder +cymbal +cynicism +cypress +cytoplasm +dachshund +daffodil +dagger +dairy +dalmatian +dandelion +dartboard +dastardly +datebook +daughter +dawn +daytime +dazzler +dealer +debris +decal +dedicate +deepness +defrost +degree +dehydrator +deliverer +democrat +dentist +deodorant +depot +deranged +desktop +detergent +device +dexterity +diamond +dibs +dictionary +diffuser +digit +dilated +dimple +dinnerware +dioxide +diploma +directory +dishcloth +ditto +dividers +dizziness +doctor +dodge +doll +dominoes +donut +doorstep +dorsal +double +downstairs +dozed +drainpipe +dresser +driftwood +droppings +drum +dryer +dubiously +duckling +duffel +dugout +dumpster +duplex +durable +dustpan +dutiful +duvet +dwarfism +dwelling +dwindling +dynamite +dyslexia +eagerness +earlobe +easel +eavesdrop +ebook +eccentric +echoless +eclipse +ecosystem +ecstasy +edged +editor +educator +eelworm +eerie +effects +eggnog +egomaniac +ejection +elastic +elbow +elderly +elephant +elfishly +eliminator +elk +elliptical +elongated +elsewhere +elusive +elves +emancipate +embroidery +emcee +emerald +emission +emoticon +emperor +emulate +enactment +enchilada +endorphin +energy +enforcer +engine +enhance +enigmatic +enjoyably +enlarged +enormous +enquirer +enrollment +ensemble +entryway +enunciate +envoy +enzyme +epidemic +equipment +erasable +ergonomic +erratic +eruption +escalator +eskimo +esophagus +espresso +essay +estrogen +etching +eternal +ethics +etiquette +eucalyptus +eulogy +euphemism +euthanize +evacuation +evergreen +evidence +evolution +exam +excerpt +exerciser +exfoliate +exhale +exist +exorcist +explode +exquisite +exterior +exuberant +fabric +factory +faded +failsafe +falcon +family +fanfare +fasten +faucet +favorite +feasibly +february +federal +feedback +feigned +feline +femur +fence +ferret +festival +fettuccine +feudalist +feverish +fiberglass +fictitious +fiddle +figurine +fillet +finalist +fiscally +fixture +flashlight +fleshiness +flight +florist +flypaper +foamless +focus +foggy +folksong +fondue +footpath +fossil +fountain +fox +fragment +freeway +fridge +frosting +fruit +fryingpan +gadget +gainfully +gallstone +gamekeeper +gangway +garlic +gaslight +gathering +gauntlet +gearbox +gecko +gem +generator +geographer +gerbil +gesture +getaway +geyser +ghoulishly +gibberish +giddiness +giftshop +gigabyte +gimmick +giraffe +giveaway +gizmo +glasses +gleeful +glisten +glove +glucose +glycerin +gnarly +gnomish +goatskin +goggles +goldfish +gong +gooey +gorgeous +gosling +gothic +gourmet +governor +grape +greyhound +grill +groundhog +grumbling +guacamole +guerrilla +guitar +gullible +gumdrop +gurgling +gusto +gutless +gymnast +gynecology +gyration +habitat +hacking +haggard +haiku +halogen +hamburger +handgun +happiness +hardhat +hastily +hatchling +haughty +hazelnut +headband +hedgehog +hefty +heinously +helmet +hemoglobin +henceforth +herbs +hesitation +hexagon +hubcap +huddling +huff +hugeness +hullabaloo +human +hunter +hurricane +hushing +hyacinth +hybrid +hydrant +hygienist +hypnotist +ibuprofen +icepack +icing +iconic +identical +idiocy +idly +igloo +ignition +iguana +illuminate +imaging +imbecile +imitator +immigrant +imprint +iodine +ionosphere +ipad +iphone +iridescent +irksome +iron +irrigation +island +isotope +issueless +italicize +itemizer +itinerary +itunes +ivory +jabbering +jackrabbit +jaguar +jailhouse +jalapeno +jamboree +janitor +jarring +jasmine +jaundice +jawbreaker +jaywalker +jazz +jealous +jeep +jelly +jeopardize +jersey +jetski +jezebel +jiffy +jigsaw +jingling +jobholder +jockstrap +jogging +john +joinable +jokingly +journal +jovial +joystick +jubilant +judiciary +juggle +juice +jujitsu +jukebox +jumpiness +junkyard +juror +justifying +juvenile +kabob +kamikaze +kangaroo +karate +kayak +keepsake +kennel +kerosene +ketchup +khaki +kickstand +kilogram +kimono +kingdom +kiosk +kissing +kite +kleenex +knapsack +kneecap +knickers +koala +krypton +laboratory +ladder +lakefront +lantern +laptop +laryngitis +lasagna +latch +laundry +lavender +laxative +lazybones +lecturer +leftover +leggings +leisure +lemon +length +leopard +leprechaun +lettuce +leukemia +levers +lewdness +liability +library +licorice +lifeboat +lightbulb +likewise +lilac +limousine +lint +lioness +lipstick +liquid +listless +litter +liverwurst +lizard +llama +luau +lubricant +lucidity +ludicrous +luggage +lukewarm +lullaby +lumberjack +lunchbox +luridness +luscious +luxurious +lyrics +macaroni +maestro +magazine +mahogany +maimed +majority +makeover +malformed +mammal +mango +mapmaker +marbles +massager +matchstick +maverick +maximum +mayonnaise +moaning +mobilize +moccasin +modify +moisture +molecule +momentum +monastery +moonshine +mortuary +mosquito +motorcycle +mousetrap +movie +mower +mozzarella +muckiness +mudflow +mugshot +mule +mummy +mundane +muppet +mural +mustard +mutation +myriad +myspace +myth +nail +namesake +nanosecond +napkin +narrator +nastiness +natives +nautically +navigate +nearest +nebula +nectar +nefarious +negotiator +neither +nemesis +neoliberal +nephew +nervously +nest +netting +neuron +nevermore +nextdoor +nicotine +niece +nimbleness +nintendo +nirvana +nuclear +nugget +nuisance +nullify +numbing +nuptials +nursery +nutcracker +nylon +oasis +oat +obediently +obituary +object +obliterate +obnoxious +observer +obtain +obvious +occupation +oceanic +octopus +ocular +office +oftentimes +oiliness +ointment +older +olympics +omissible +omnivorous +oncoming +onion +onlooker +onstage +onward +onyx +oomph +opaquely +opera +opium +opossum +opponent +optical +opulently +oscillator +osmosis +ostrich +otherwise +ought +outhouse +ovation +oven +owlish +oxford +oxidize +oxygen +oyster +ozone +pacemaker +padlock +pageant +pajamas +palm +pamphlet +pantyhose +paprika +parakeet +passport +patio +pauper +pavement +payphone +pebble +peculiarly +pedometer +pegboard +pelican +penguin +peony +pepperoni +peroxide +pesticide +petroleum +pewter +pharmacy +pheasant +phonebook +phrasing +physician +plank +pledge +plotted +plug +plywood +pneumonia +podiatrist +poetic +pogo +poison +poking +policeman +poncho +popcorn +porcupine +postcard +poultry +powerboat +prairie +pretzel +princess +propeller +prune +pry +pseudo +psychopath +publisher +pucker +pueblo +pulley +pumpkin +punchbowl +puppy +purse +pushup +putt +puzzle +pyramid +python +quarters +quesadilla +quilt +quote +racoon +radish +ragweed +railroad +rampantly +rancidity +rarity +raspberry +ravishing +rearrange +rebuilt +receipt +reentry +refinery +register +rehydrate +reimburse +rejoicing +rekindle +relic +remote +renovator +reopen +reporter +request +rerun +reservoir +retriever +reunion +revolver +rewrite +rhapsody +rhetoric +rhino +rhubarb +rhyme +ribbon +riches +ridden +rigidness +rimmed +riptide +riskily +ritzy +riverboat +roamer +robe +rocket +romancer +ropelike +rotisserie +roundtable +royal +rubber +rudderless +rugby +ruined +rulebook +rummage +running +rupture +rustproof +sabotage +sacrifice +saddlebag +saffron +sainthood +saltshaker +samurai +sandworm +sapphire +sardine +sassy +satchel +sauna +savage +saxophone +scarf +scenario +schoolbook +scientist +scooter +scrapbook +sculpture +scythe +secretary +sedative +segregator +seismology +selected +semicolon +senator +septum +sequence +serpent +sesame +settler +severely +shack +shelf +shirt +shovel +shrimp +shuttle +shyness +siamese +sibling +siesta +silicon +simmering +singles +sisterhood +sitcom +sixfold +sizable +skateboard +skeleton +skies +skulk +skylight +slapping +sled +slingshot +sloth +slumbering +smartphone +smelliness +smitten +smokestack +smudge +snapshot +sneezing +sniff +snowsuit +snugness +speakers +sphinx +spider +splashing +sponge +sprout +spur +spyglass +squirrel +statue +steamboat +stingray +stopwatch +strawberry +student +stylus +suave +subway +suction +suds +suffocate +sugar +suitcase +sulphur +superstore +surfer +sushi +swan +sweatshirt +swimwear +sword +sycamore +syllable +symphony +synagogue +syringes +systemize +tablespoon +taco +tadpole +taekwondo +tagalong +takeout +tallness +tamale +tanned +tapestry +tarantula +tastebud +tattoo +tavern +thaw +theater +thimble +thorn +throat +thumb +thwarting +tiara +tidbit +tiebreaker +tiger +timid +tinsel +tiptoeing +tirade +tissue +tractor +tree +tripod +trousers +trucks +tryout +tubeless +tuesday +tugboat +tulip +tumbleweed +tupperware +turtle +tusk +tutorial +tuxedo +tweezers +twins +tyrannical +ultrasound +umbrella +umpire +unarmored +unbuttoned +uncle +underwear +unevenness +unflavored +ungloved +unhinge +unicycle +unjustly +unknown +unlocking +unmarked +unnoticed +unopened +unpaved +unquenched +unroll +unscrewing +untied +unusual +unveiled +unwrinkled +unyielding +unzip +upbeat +upcountry +update +upfront +upgrade +upholstery +upkeep +upload +uppercut +upright +upstairs +uptown +upwind +uranium +urban +urchin +urethane +urgent +urologist +username +usher +utensil +utility +utmost +utopia +utterance +vacuum +vagrancy +valuables +vanquished +vaporizer +varied +vaseline +vegetable +vehicle +velcro +vendor +vertebrae +vestibule +veteran +vexingly +vicinity +videogame +viewfinder +vigilante +village +vinegar +violin +viperfish +virus +visor +vitamins +vivacious +vixen +vocalist +vogue +voicemail +volleyball +voucher +voyage +vulnerable +waffle +wagon +wakeup +walrus +wanderer +wasp +water +waving +wheat +whisper +wholesaler +wick +widow +wielder +wifeless +wikipedia +wildcat +windmill +wipeout +wired +wishbone +wizardry +wobbliness +wolverine +womb +woolworker +workbasket +wound +wrangle +wreckage +wristwatch +wrongdoing +xerox +xylophone +yacht +yahoo +yard +yearbook +yesterday +yiddish +yield +yo-yo +yodel +yogurt +yuppie +zealot +zebra +zeppelin +zestfully +zigzagged +zillion +zipping +zirconium +zodiac +zombie +zookeeper +zucchini diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/PermissionTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/PermissionTest.java index 02a9995905..a6ca3d9906 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/PermissionTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/PermissionTest.java @@ -60,13 +60,13 @@ import com.linecorp.centraldogma.server.command.StandaloneCommandExecutor; import com.linecorp.centraldogma.server.internal.api.HttpApiExceptionHandler; import com.linecorp.centraldogma.server.internal.storage.project.DefaultProjectManager; -import com.linecorp.centraldogma.server.internal.storage.project.ProjectInitializer; import com.linecorp.centraldogma.server.management.ServerStatusManager; import com.linecorp.centraldogma.server.metadata.MetadataService; import com.linecorp.centraldogma.server.metadata.MetadataServiceInjector; import com.linecorp.centraldogma.server.metadata.PerRolePermissions; import com.linecorp.centraldogma.server.metadata.Permission; import com.linecorp.centraldogma.server.metadata.ProjectRole; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import com.linecorp.centraldogma.testing.internal.TemporaryFolderExtension; @@ -97,8 +97,7 @@ protected void configure(ServerBuilder sb) throws Exception { final CommandExecutor executor = new StandaloneCommandExecutor( pm, ForkJoinPool.commonPool(), statusManager, null, null, null); executor.start().join(); - - ProjectInitializer.initializeInternalProject(executor); + new InternalProjectInitializer(executor).initialize(); executor.execute(Command.createProject(AUTHOR, "project1")).join(); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirrorTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirrorTest.java index 105d43d92b..9133b605c9 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirrorTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirrorTest.java @@ -22,8 +22,6 @@ import java.net.URI; -import javax.annotation.Nullable; - import org.junit.jupiter.api.Test; import com.cronutils.model.Cron; @@ -50,31 +48,31 @@ void testCentralDogmaMirror() { // Simplest possible form m = assertMirror("dogma://a.com/b/c.dogma", CentralDogmaMirror.class, - "dogma://a.com/b/c.dogma", "/", null); + "dogma://a.com/b/c.dogma", "/", ""); assertThat(m.remoteProject()).isEqualTo("b"); assertThat(m.remoteRepo()).isEqualTo("c"); // Non-default port number m = assertMirror("dogma://a.com:1234/b/c.dogma", CentralDogmaMirror.class, - "dogma://a.com:1234/b/c.dogma", "/", null); + "dogma://a.com:1234/b/c.dogma", "/", ""); assertThat(m.remoteProject()).isEqualTo("b"); assertThat(m.remoteRepo()).isEqualTo("c"); // Non-default remotePath m = assertMirror("dogma://a.com/b/c.dogma/d", CentralDogmaMirror.class, - "dogma://a.com/b/c.dogma", "/d/", null); + "dogma://a.com/b/c.dogma", "/d/", ""); assertThat(m.remoteProject()).isEqualTo("b"); assertThat(m.remoteRepo()).isEqualTo("c"); // Non-default remoteBranch (should be ignored because Central Dogma has no notion of branch.) m = assertMirror("dogma://a.com/b/c.dogma#develop", CentralDogmaMirror.class, - "dogma://a.com/b/c.dogma", "/", null); + "dogma://a.com/b/c.dogma", "/", ""); assertThat(m.remoteProject()).isEqualTo("b"); assertThat(m.remoteRepo()).isEqualTo("c"); // Non-default remotePath and remoteBranch m = assertMirror("dogma://a.com/b/c.dogma/d#develop", CentralDogmaMirror.class, - "dogma://a.com/b/c.dogma", "/d/", null); + "dogma://a.com/b/c.dogma", "/d/", ""); assertThat(m.remoteProject()).isEqualTo("b"); assertThat(m.remoteRepo()).isEqualTo("c"); @@ -100,7 +98,7 @@ void testUnknownScheme() { private static T assertMirror(String remoteUri, Class mirrorType, String expectedRemoteRepoUri, String expectedRemotePath, - @Nullable String expectedRemoteBranch) { + String expectedRemoteBranch) { final Repository repository = mock(Repository.class); final Project project = mock(Project.class); when(repository.parent()).thenReturn(project); @@ -121,12 +119,14 @@ static T newMirror(String remoteUri, Class mirrorType) { static T newMirror(String remoteUri, Cron schedule, Repository repository, Class mirrorType) { final MirrorCredential credential = mock(MirrorCredential.class); + final String mirrorId = "mirror-id"; final Mirror mirror = new CentralDogmaMirrorProvider().newMirror( - new MirrorContext(schedule, MirrorDirection.LOCAL_TO_REMOTE, + new MirrorContext(mirrorId, true, schedule, MirrorDirection.LOCAL_TO_REMOTE, credential, repository, "/", URI.create(remoteUri), null)); assertThat(mirror).isInstanceOf(mirrorType); + assertThat(mirror.id()).isEqualTo(mirrorId); assertThat(mirror.direction()).isEqualTo(MirrorDirection.LOCAL_TO_REMOTE); assertThat(mirror.credential()).isSameAs(credential); assertThat(mirror.localRepo()).isSameAs(repository); @@ -140,7 +140,7 @@ static T newMirror(String remoteUri, Cron schedule, static void assertMirrorNull(String remoteUri) { final MirrorCredential credential = mock(MirrorCredential.class); final Mirror mirror = new CentralDogmaMirrorProvider().newMirror( - new MirrorContext(EVERY_MINUTE, MirrorDirection.LOCAL_TO_REMOTE, + new MirrorContext("mirror-id", true, EVERY_MINUTE, MirrorDirection.LOCAL_TO_REMOTE, credential, mock(Repository.class), "/", URI.create(remoteUri), null)); assertThat(mirror).isNull(); } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/AccessTokenMirrorCredentialTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/AccessTokenMirrorCredentialTest.java index 2644c89c22..f144aecf25 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/AccessTokenMirrorCredentialTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/AccessTokenMirrorCredentialTest.java @@ -30,15 +30,16 @@ class AccessTokenMirrorCredentialTest { @Test void testConstruction() throws Exception { // null checks - assertThatThrownBy(() -> new AccessTokenMirrorCredential(null, null, null)) + assertThatThrownBy(() -> new AccessTokenMirrorCredential("foo", true, null, null)) .isInstanceOf(NullPointerException.class); // emptiness checks - assertThatThrownBy(() -> new AccessTokenMirrorCredential(null, null, "")) + assertThatThrownBy(() -> new AccessTokenMirrorCredential("foo", true, null, "")) .isInstanceOf(IllegalArgumentException.class); // successful construction - final AccessTokenMirrorCredential c = new AccessTokenMirrorCredential(null, null, "sesame"); + final AccessTokenMirrorCredential c = new AccessTokenMirrorCredential("foo", true, null, "sesame"); + assertThat(c.id()).isEqualTo("foo"); assertThat(c.accessToken()).isEqualTo("sesame"); } @@ -46,20 +47,21 @@ void testConstruction() throws Exception { void testDeserialization() throws Exception { // With hostnamePatterns assertThat(Jackson.readValue('{' + + " \"id\": \"access-token-id\"," + " \"type\": \"access_token\"," + " \"hostnamePatterns\": [" + " \"^foo\\\\.com$\"" + " ]," + " \"accessToken\": \"sesame\"" + '}', MirrorCredential.class)) - .isEqualTo(new AccessTokenMirrorCredential(null, HOSTNAME_PATTERNS, + .isEqualTo(new AccessTokenMirrorCredential("access-token-id", true, HOSTNAME_PATTERNS, "sesame")); - // With ID + // Without hostnamePatterns assertThat(Jackson.readValue('{' + " \"type\": \"access_token\"," + " \"id\": \"foo\"," + " \"accessToken\": \"sesame\"" + '}', MirrorCredential.class)) - .isEqualTo(new AccessTokenMirrorCredential("foo", null, "sesame")); + .isEqualTo(new AccessTokenMirrorCredential("foo", true, null, "sesame")); } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/MirrorCredentialTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/MirrorCredentialTest.java index 89b0e587f6..6221724ff0 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/MirrorCredentialTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/MirrorCredentialTest.java @@ -43,16 +43,15 @@ class MirrorCredentialTest { @Test void testConstruction() { - // Without ID and hostnamePatterns, i.e. effectively disabled. - assertThat(new MirrorCredentialImpl(null, null).id()).isEmpty(); - assertThat(new MirrorCredentialImpl(null, null).hostnamePatterns()).isEmpty(); + // null hostnamePatterns. + assertThat(new MirrorCredentialImpl("foo", null).hostnamePatterns()).isEmpty(); - // Without ID and with hostnamePatterns that contain null. - assertThatThrownBy(() -> new MirrorCredentialImpl(null, INVALID_PATTERNS)) + // hostnamePatterns that contain null. + assertThatThrownBy(() -> new MirrorCredentialImpl("foo", INVALID_PATTERNS)) .isInstanceOf(NullPointerException.class); - // Without ID and with an empty hostnamePatterns. - assertThat(new MirrorCredentialImpl(null, ImmutableSet.of()).hostnamePatterns()).isEmpty(); + // An empty hostnamePatterns. + assertThat(new MirrorCredentialImpl("foo", ImmutableSet.of()).hostnamePatterns()).isEmpty(); // With ID and non-empty hostnamePatterns. final MirrorCredential c = new MirrorCredentialImpl("foo", HOSTNAME_PATTERNS); @@ -62,8 +61,8 @@ void testConstruction() { } private static final class MirrorCredentialImpl extends AbstractMirrorCredential { - MirrorCredentialImpl(@Nullable String id, @Nullable Iterable hostnamePatterns) { - super(id, hostnamePatterns); + MirrorCredentialImpl(String id, @Nullable Iterable hostnamePatterns) { + super(id, true, "custom", hostnamePatterns); } @Override diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/NoneMirrorCredentialTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/NoneMirrorCredentialTest.java index e29e1642c4..60555ca49e 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/NoneMirrorCredentialTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/NoneMirrorCredentialTest.java @@ -33,17 +33,21 @@ class NoneMirrorCredentialTest { void testDeserialization() throws Exception { // With hostnamePatterns assertThat(Jackson.readValue('{' + + " \"id\": \"none\"," + " \"type\": \"none\"," + " \"hostnamePatterns\": [" + " \"^foo\\\\.com$\"" + - " ]" + + " ]," + + " \"enabled\": true" + '}', MirrorCredential.class)) - .isEqualTo(new NoneMirrorCredential(null, ImmutableSet.of(Pattern.compile("^foo\\.com$")))); - // With ID + .isEqualTo(new NoneMirrorCredential("none", true, + ImmutableSet.of(Pattern.compile("^foo\\.com$")) + )); + // Without hostnamePatterns assertThat(Jackson.readValue('{' + " \"type\": \"none\"," + " \"id\": \"foo\"" + '}', MirrorCredential.class)) - .isEqualTo(new NoneMirrorCredential("foo", null)); + .isEqualTo(new NoneMirrorCredential("foo", true, null)); } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/PasswordMirrorCredentialTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/PasswordMirrorCredentialTest.java index 041ed03d98..b4669acef7 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/PasswordMirrorCredentialTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/PasswordMirrorCredentialTest.java @@ -30,21 +30,22 @@ class PasswordMirrorCredentialTest { @Test void testConstruction() throws Exception { // null checks - assertThatThrownBy(() -> new PasswordMirrorCredential(null, null, null, "sesame")) + assertThatThrownBy(() -> new PasswordMirrorCredential("foo", true, null, null, "sesame")) .isInstanceOf(NullPointerException.class); - assertThatThrownBy(() -> new PasswordMirrorCredential(null, null, "trustin", null)) + assertThatThrownBy(() -> new PasswordMirrorCredential("foo", true, null, "trustin", null)) .isInstanceOf(NullPointerException.class); // emptiness checks - assertThatThrownBy(() -> new PasswordMirrorCredential(null, null, "", "sesame")) + assertThatThrownBy(() -> new PasswordMirrorCredential("foo", true, null, "", "sesame")) .isInstanceOf(IllegalArgumentException.class); // An empty password must be allowed because some servers uses password authentication // as token-based authentication whose username is the token and password is an empty string. - assertThat(new PasswordMirrorCredential(null, null, "trustin", "").password()).isEmpty(); + assertThat(new PasswordMirrorCredential("foo", true, null, "trustin", "").password()).isEmpty(); // successful construction - final PasswordMirrorCredential c = new PasswordMirrorCredential(null, null, "trustin", "sesame"); + final PasswordMirrorCredential c = new PasswordMirrorCredential("foo", true, null, "trustin", "sesame"); + assertThat(c.id()).isEqualTo("foo"); assertThat(c.username()).isEqualTo("trustin"); assertThat(c.password()).isEqualTo("sesame"); } @@ -53,6 +54,7 @@ void testConstruction() throws Exception { void testDeserialization() throws Exception { // With hostnamePatterns assertThat(Jackson.readValue('{' + + " \"id\": \"password-id\"," + " \"type\": \"password\"," + " \"hostnamePatterns\": [" + " \"^foo\\\\.com$\"" + @@ -60,15 +62,15 @@ void testDeserialization() throws Exception { " \"username\": \"trustin\"," + " \"password\": \"sesame\"" + '}', MirrorCredential.class)) - .isEqualTo(new PasswordMirrorCredential(null, HOSTNAME_PATTERNS, + .isEqualTo(new PasswordMirrorCredential("password-id", true, HOSTNAME_PATTERNS, "trustin", "sesame")); - // With ID + // Without hostnamePatterns assertThat(Jackson.readValue('{' + " \"type\": \"password\"," + " \"id\": \"foo\"," + " \"username\": \"trustin\"," + " \"password\": \"sesame\"" + '}', MirrorCredential.class)) - .isEqualTo(new PasswordMirrorCredential("foo", null, "trustin", "sesame")); + .isEqualTo(new PasswordMirrorCredential("foo", true, null, "trustin", "sesame")); } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/PublicKeyMirrorCredentialTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/PublicKeyMirrorCredentialTest.java index 87114d351a..388c781409 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/PublicKeyMirrorCredentialTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/mirror/credential/PublicKeyMirrorCredentialTest.java @@ -24,14 +24,13 @@ import org.junit.jupiter.api.Test; -import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.server.ConfigValueConverter; import com.linecorp.centraldogma.server.mirror.MirrorCredential; -class PublicKeyMirrorCredentialTest { +public class PublicKeyMirrorCredentialTest { private static final String USERNAME = "trustin"; @@ -68,51 +67,48 @@ class PublicKeyMirrorCredentialTest { void testConstruction() { // null checks assertThatThrownBy(() -> new PublicKeyMirrorCredential( - null, null, null, PUBLIC_KEY, PRIVATE_KEY, PASSPHRASE)) + "id", true, null, null, PUBLIC_KEY, PRIVATE_KEY, PASSPHRASE)) .isInstanceOf(NullPointerException.class); assertThatThrownBy(() -> new PublicKeyMirrorCredential( - null, null, USERNAME, null, PRIVATE_KEY, PASSPHRASE)) + "id", true, null, USERNAME, null, PRIVATE_KEY, PASSPHRASE)) .isInstanceOf(NullPointerException.class); assertThatThrownBy(() -> new PublicKeyMirrorCredential( - null, null, USERNAME, PUBLIC_KEY, null, PASSPHRASE)) + "id", true, null, USERNAME, PUBLIC_KEY, null, PASSPHRASE)) .isInstanceOf(NullPointerException.class); // null passphrase must be accepted. assertThat(new PublicKeyMirrorCredential( - null, null, USERNAME, PUBLIC_KEY, PRIVATE_KEY, null).passphrase()).isNull(); + "id", true, null, USERNAME, PUBLIC_KEY, PRIVATE_KEY, null).passphrase()).isNull(); // emptiness checks assertThatThrownBy(() -> new PublicKeyMirrorCredential( - null, null, "", PUBLIC_KEY, PRIVATE_KEY, PASSPHRASE)) + "id", true, null, "", PUBLIC_KEY, PRIVATE_KEY, PASSPHRASE)) .isInstanceOf(IllegalArgumentException.class); assertThatThrownBy(() -> new PublicKeyMirrorCredential( - null, null, USERNAME, "", PRIVATE_KEY, PASSPHRASE)) + "id", true, null, USERNAME, "", PRIVATE_KEY, PASSPHRASE)) .isInstanceOf(IllegalArgumentException.class); assertThatThrownBy(() -> new PublicKeyMirrorCredential( - null, null, USERNAME, PUBLIC_KEY, "", PASSPHRASE)) + "id", true, null, USERNAME, PUBLIC_KEY, "", PASSPHRASE)) .isInstanceOf(IllegalArgumentException.class); // empty passphrase must be accepted, because an empty password is still a password. assertThat(new PublicKeyMirrorCredential( - null, null, USERNAME, PUBLIC_KEY, PRIVATE_KEY, "").passphrase()).isEmpty(); + "id", true, null, USERNAME, PUBLIC_KEY, PRIVATE_KEY, "").passphrase()).isEmpty(); // successful construction final PublicKeyMirrorCredential c = new PublicKeyMirrorCredential( - null, null, USERNAME, PUBLIC_KEY, PRIVATE_KEY, PASSPHRASE); + "id", true, null, USERNAME, PUBLIC_KEY, PRIVATE_KEY, PASSPHRASE); assertThat(c.username()).isEqualTo(USERNAME); assertThat(c.publicKey()).isEqualTo(PUBLIC_KEY); - assertThat(c.privateKey()).isEqualTo(Splitter.on('\n') - .omitEmptyStrings() - .trimResults() - .splitToList(PRIVATE_KEY)); + assertThat(c.rawPrivateKey()).isEqualTo(PRIVATE_KEY); assertThat(c.passphrase()).isEqualTo(PASSPHRASE); } @Test void testBase64Passphrase() { final PublicKeyMirrorCredential c = new PublicKeyMirrorCredential( - null, null, USERNAME, PUBLIC_KEY, PRIVATE_KEY, PASSPHRASE_BASE64); + "id", true, null, USERNAME, PUBLIC_KEY, PRIVATE_KEY, PASSPHRASE_BASE64); assertThat(c.passphrase()).isEqualTo(PASSPHRASE); } @@ -120,6 +116,7 @@ void testBase64Passphrase() { void testDeserialization() throws Exception { // plaintext passphrase assertThat(Jackson.readValue('{' + + " \"id\": \"foo\"," + " \"type\": \"public_key\"," + " \"hostnamePatterns\": [" + " \"^foo\\\\.com$\"" + @@ -129,14 +126,15 @@ void testDeserialization() throws Exception { " \"privateKey\": \"" + Jackson.escapeText(PRIVATE_KEY) + "\"," + " \"passphrase\": \"" + Jackson.escapeText(PASSPHRASE) + '"' + '}', MirrorCredential.class)) - .isEqualTo(new PublicKeyMirrorCredential(null, HOSTNAME_PATTERNS, USERNAME, + .isEqualTo(new PublicKeyMirrorCredential("foo", true, HOSTNAME_PATTERNS, USERNAME, PUBLIC_KEY, PRIVATE_KEY, PASSPHRASE)); // base64 passphrase final PublicKeyMirrorCredential base64Expected = - new PublicKeyMirrorCredential(null, HOSTNAME_PATTERNS, USERNAME, + new PublicKeyMirrorCredential("bar", null, HOSTNAME_PATTERNS, USERNAME, PUBLIC_KEY, PRIVATE_KEY, PASSPHRASE_BASE64); assertThat(Jackson.readValue('{' + + " \"id\": \"bar\"," + " \"type\": \"public_key\"," + " \"hostnamePatterns\": [" + " \"^foo\\\\.com$\"" + @@ -149,7 +147,7 @@ void testDeserialization() throws Exception { .isEqualTo(base64Expected); assertThat(base64Expected.passphrase()).isEqualTo(PASSPHRASE); - // ID + // Without hostnamePatterns assertThat(Jackson.readValue('{' + " \"type\": \"public_key\"," + " \"id\": \"foo\"," + @@ -158,11 +156,11 @@ void testDeserialization() throws Exception { " \"privateKey\": \"" + Jackson.escapeText(PRIVATE_KEY) + "\"," + " \"passphrase\": \"" + Jackson.escapeText(PASSPHRASE) + '"' + '}', MirrorCredential.class)) - .isEqualTo(new PublicKeyMirrorCredential("foo", null, USERNAME, + .isEqualTo(new PublicKeyMirrorCredential("foo", true, null, USERNAME, PUBLIC_KEY, PRIVATE_KEY, PASSPHRASE)); - final PublicKeyMirrorCredential conveterExpected = - new PublicKeyMirrorCredential("foo", null, USERNAME, + final PublicKeyMirrorCredential converterExpected = + new PublicKeyMirrorCredential("foo", true, null, USERNAME, PUBLIC_KEY, PRIVATE_KEY, "mirror_encryption:foo"); assertThat(Jackson.readValue('{' + " \"type\": \"public_key\"," + @@ -172,8 +170,8 @@ void testDeserialization() throws Exception { " \"privateKey\": \"" + Jackson.escapeText(PRIVATE_KEY) + "\"," + " \"passphrase\": \"mirror_encryption:foo\"" + '}', MirrorCredential.class)) - .isEqualTo(conveterExpected); - assertThat(conveterExpected.passphrase()).isEqualTo("bar"); + .isEqualTo(converterExpected); + assertThat(converterExpected.passphrase()).isEqualTo("bar"); } public static class PasswordConfigValueConverter implements ConfigValueConverter { diff --git a/site/build.gradle b/site/build.gradle index f821712117..fcb30e5ff7 100644 --- a/site/build.gradle +++ b/site/build.gradle @@ -5,10 +5,16 @@ import java.util.stream.Collectors plugins { id 'base' - alias libs.plugins.sphinx + alias libs.plugins.sphinx apply false alias libs.plugins.jxr } +if (project.hasProperty("noSite")) { + return +} + +apply plugin: "kr.motd.sphinx" + sphinx { group = 'Documentation' description = 'Generates the Sphinx web site.' diff --git a/site/src/sphinx/mirroring.rst b/site/src/sphinx/mirroring.rst index 6d844d9c13..41da90b2b2 100644 --- a/site/src/sphinx/mirroring.rst +++ b/site/src/sphinx/mirroring.rst @@ -34,28 +34,37 @@ applications access Central Dogma repositories instead: Setting up a Git-to-CD mirror ----------------------------- -You need to put two files into the ``meta`` repository of your Central Dogma project: ``/mirrors.json`` and -``/credentials.json``. +You need to put two files into the ``meta`` repository of your Central Dogma project: +``/mirrors/{mirror-id}.json`` and ``/credentials/{credential-id}.json``. -``/mirrors.json`` contains an array of periodic mirroring tasks. For example: +Setting up a mirroring task +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: json +``/mirrors/{mirror-id}.json`` contains an object of a periodic mirroring task. For example: - [ - { - "enabled": true, - "schedule": "0 * * * * ?", - "direction": "REMOTE_TO_LOCAL", - "localRepo": "foo", - "localPath": "/", - "remoteUri": "git+ssh://git.example.com/foo.git/settings#release", - "credentialId": "my_private_key", - "gitignore": [ - "/credential.txt", - "private_dir" - ] - } - ] +.. code-block:: json + :caption: foo-settings-mirror.json + + { + "id": "foo-settings-mirror", + "enabled": true, + "schedule": "0 * * * * ?", + "direction": "REMOTE_TO_LOCAL", + "localRepo": "foo", + "localPath": "/", + "remoteUri": "git+ssh://git.example.com/foo.git/settings#release", + "credentialId": "my_private_key", + "gitignore": [ + "/credential.txt", + "private_dir" + ] + } + +- ``id`` (string) + + - the ID of the mirroring task. You should set the same value specified in the file name. + For example, if the file name is ``/mirrors/foo-settings-mirror.json``, the value of this field should be + ``foo-settings-mirror``. - ``enabled`` (boolean, optional) @@ -99,9 +108,9 @@ You need to put two files into the ``meta`` repository of your Central Dogma pro - ``credentialId`` (string, optional) - - the ID of the credential to use for authentication, as defined in ``/credentials.json``. If unspecified, - the credential whose ``hostnamePattern`` is matched by the host name part of the ``remoteUri`` value will - be selected automatically. + - the ID of the credential to use for authentication, as defined in ``/credentials/{credential-id}.json``. + If unspecified, the credential whose ``hostnamePattern`` is matched by the host name part of the + ``remoteUri`` value will be selected automatically. - ``gitignore`` (string or array of strings, optional) @@ -110,48 +119,70 @@ You need to put two files into the ``meta`` repository of your Central Dogma pro of strings where each line represents a single pattern. The file pattern expressed in gitignore is relative to the path of ``remoteUri``. -``/credentials.json`` contains the authentication credentials which are required when accessing the Git -repositories defined in ``/mirrors.json``: +Setting up a credential +^^^^^^^^^^^^^^^^^^^^^^^ + +``/credentials/{credential-id}.json`` contains the authentication credential which is required when accessing +the Git repositories defined in ``/mirrors/{mirror-id}.json``: + +* No authentication +.. code-block:: json + :caption: no_auth.json + + { + "id": "no_auth", + "type": "none", + "hostnamePatterns": [ + "^git\.insecure\.com$" + ] + } +* Password-based authentication +.. code-block:: json + :caption: my_password.json + + { + "id": "my_password", + "type": "password", + "hostnamePatterns": [ + "^git\.password-protected\.com$" + ], + "username": "alice", + "password": "secret!" + } + +* SSH public key authentication +.. code-block:: json + :caption: my_private_key.json + + { + "id": "my_private_key", + "type": "public_key", + "hostnamePatterns": [ + "^.*\.secure\.com$" + ], + "username": "git", + "publicKey": "ssh-ed25519 ... user@host", + "privateKey": "-----BEGIN OPENSSH PRIVATE KEY-----\n...\n-----END OPENSSH PRIVATE KEY-----\n", + "passphrase": null + } + +* Access token-based authentication .. code-block:: json + :caption: my_access_token.json + + { + "id": "my_access_token", + "type": "access_token", + "accessToken": "github_pat_..." + } + +- ``id`` (string) - [ - { - "type": "none", - "hostnamePatterns": [ - "^git\.insecure\.com$" - ] - }, - { - "type": "password", - "hostnamePatterns": [ - "^git\.password-protected\.com$" - ], - "username": "alice", - "password": "secret!" - }, - { - "id": "my_private_key", - "type": "public_key", - "hostnamePatterns": [ - "^.*\.secure\.com$" - ], - "username": "git", - "publicKey": "ssh-ed25519 ... user@host", - "privateKey": "-----BEGIN OPENSSH PRIVATE KEY-----\n...\n-----END OPENSSH PRIVATE KEY-----\n", - "passphrase": null - }, - { - "id": "my_access_token", - "type": "access_token", - "accessToken": "github_pat_..." - } - ] - -- ``id`` (string, optional) - - - the ID of the credential. You can specify the value of this field in the ``credentialId`` field of the - mirror definitions in ``/mirrors.json``. + - the ID of the credential. You should set the same value specified in the file name. For example, if the file + name is ``/credentials/my_private_key.json``, the value of this field should be ``my_private_key``. + You can specify the value of this field in the ``credentialId`` field of the mirror definitions in + ``/mirrors/{mirror-id}.json``. - ``type`` (string) @@ -159,7 +190,7 @@ repositories defined in ``/mirrors.json``: - ``hostnamePatterns`` (array of strings, optional) - - the regular repressions that matches a host name. The credential whose hostname pattern matches first will + - the regular expression that matches a host name. The credential whose hostname pattern matches first will be used when accessing a host. You may want to omit this field if you do not want the credential to be selected automatically, i.e. a mirror has to specify the ``credentialId`` field. diff --git a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/ProjectManagerExtension.java b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/ProjectManagerExtension.java index 12a139a10d..c070007783 100644 --- a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/ProjectManagerExtension.java +++ b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/ProjectManagerExtension.java @@ -30,8 +30,8 @@ import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.command.StandaloneCommandExecutor; import com.linecorp.centraldogma.server.internal.storage.project.DefaultProjectManager; -import com.linecorp.centraldogma.server.internal.storage.project.ProjectInitializer; import com.linecorp.centraldogma.server.management.ServerStatusManager; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import com.linecorp.centraldogma.testing.junit.AbstractAllOrEachExtension; @@ -58,6 +58,7 @@ public class ProjectManagerExtension extends AbstractAllOrEachExtension { private ProjectManager projectManager; private CommandExecutor executor; private ScheduledExecutorService purgeWorker; + private InternalProjectInitializer internalProjectInitializer; private final TemporaryFolder tempDir = new TemporaryFolder(); private File dataDir; @@ -77,7 +78,8 @@ public void before(ExtensionContext context) throws Exception { executor = newCommandExecutor(projectManager, repositoryWorker, dataDir); executor.start().get(); - ProjectInitializer.initializeInternalProject(executor); + internalProjectInitializer = new InternalProjectInitializer(executor); + internalProjectInitializer.initialize(); afterExecutorStarted(); } diff --git a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/auth/TestAuthMessageUtil.java b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/auth/TestAuthMessageUtil.java index 3196823eda..a2029f9f8e 100644 --- a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/auth/TestAuthMessageUtil.java +++ b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/auth/TestAuthMessageUtil.java @@ -15,16 +15,23 @@ */ package com.linecorp.centraldogma.testing.internal.auth; +import static org.assertj.core.api.Assertions.assertThat; + import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Base64.Encoder; +import com.fasterxml.jackson.core.JsonProcessingException; + import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.HttpHeaderNames; import com.linecorp.armeria.common.HttpMethod; +import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.RequestHeaders; +import com.linecorp.centraldogma.internal.Jackson; +import com.linecorp.centraldogma.internal.api.v1.AccessToken; /** * A utility class which helps to create messages for authentication. @@ -51,11 +58,10 @@ public static AggregatedHttpResponse login(WebClient client, String username, St public static AggregatedHttpResponse loginWithBasicAuth(WebClient client, String username, String password) { - return client.execute( - RequestHeaders.of(HttpMethod.POST, "/api/v1/login", - HttpHeaderNames.AUTHORIZATION, - "basic " + encoder.encodeToString( - (username + ':' + password).getBytes(StandardCharsets.US_ASCII)))) + final String token = "basic " + encoder.encodeToString( + (username + ':' + password).getBytes(StandardCharsets.US_ASCII)); + return client.execute(RequestHeaders.of(HttpMethod.POST, "/api/v1/login", + HttpHeaderNames.AUTHORIZATION, token)) .aggregate().join(); } @@ -71,5 +77,13 @@ public static AggregatedHttpResponse usersMe(WebClient client, String sessionId) HttpHeaderNames.AUTHORIZATION, "Bearer " + sessionId)).aggregate().join(); } + public static String getAccessToken(WebClient client, String username, String password) + throws JsonProcessingException { + final AggregatedHttpResponse response = login(client, username, password); + assertThat(response.status()).isEqualTo(HttpStatus.OK); + return Jackson.readValue(response.content().array(), AccessToken.class) + .accessToken(); + } + private TestAuthMessageUtil() {} } diff --git a/testing/common/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaRuleDelegate.java b/testing/common/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaRuleDelegate.java index a7797ad088..e4e55f79b6 100644 --- a/testing/common/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaRuleDelegate.java +++ b/testing/common/src/main/java/com/linecorp/centraldogma/testing/internal/CentralDogmaRuleDelegate.java @@ -26,6 +26,7 @@ import com.google.common.collect.Iterables; +import com.linecorp.armeria.client.BlockingWebClient; import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.client.WebClientBuilder; @@ -247,6 +248,15 @@ public final WebClient httpClient() { return webClient; } + /** + * Returns the blocking HTTP client. + * + * @throws IllegalStateException if Central Dogma did not start yet + */ + public final BlockingWebClient blockingHttpClient() { + return httpClient().blocking(); + } + /** * Returns the server address. * diff --git a/testing/junit/src/main/java/com/linecorp/centraldogma/testing/junit/CentralDogmaExtension.java b/testing/junit/src/main/java/com/linecorp/centraldogma/testing/junit/CentralDogmaExtension.java index 9597d72991..35517df6be 100644 --- a/testing/junit/src/main/java/com/linecorp/centraldogma/testing/junit/CentralDogmaExtension.java +++ b/testing/junit/src/main/java/com/linecorp/centraldogma/testing/junit/CentralDogmaExtension.java @@ -26,6 +26,7 @@ import com.spotify.futures.CompletableFutures; +import com.linecorp.armeria.client.BlockingWebClient; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.client.WebClientBuilder; import com.linecorp.armeria.common.annotation.UnstableApi; @@ -230,6 +231,15 @@ public final WebClient httpClient() { return delegate.httpClient(); } + /** + * Returns the blocking HTTP client. + * + * @throws IllegalStateException if Central Dogma did not start yet + */ + public final BlockingWebClient blockingHttpClient() { + return delegate.blockingHttpClient(); + } + /** * Returns the server address. * diff --git a/xds/src/main/java/com/linecorp/centraldogma/xds/internal/ControlPlanePlugin.java b/xds/src/main/java/com/linecorp/centraldogma/xds/internal/ControlPlanePlugin.java index 651ea41639..51753a1972 100644 --- a/xds/src/main/java/com/linecorp/centraldogma/xds/internal/ControlPlanePlugin.java +++ b/xds/src/main/java/com/linecorp/centraldogma/xds/internal/ControlPlanePlugin.java @@ -16,8 +16,7 @@ package com.linecorp.centraldogma.xds.internal; -import static com.linecorp.centraldogma.server.internal.storage.project.ProjectInitializer.INTERNAL_PROJECT_DOGMA; -import static com.linecorp.centraldogma.server.internal.storage.project.ProjectInitializer.initializeInternalRepos; +import static com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer.INTERNAL_PROJECT_DOGMA; import java.util.Collection; import java.util.Map; @@ -40,10 +39,10 @@ import com.linecorp.armeria.server.grpc.GrpcService; import com.linecorp.centraldogma.common.Entry; import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.plugin.AllReplicasPlugin; import com.linecorp.centraldogma.server.plugin.PluginContext; import com.linecorp.centraldogma.server.plugin.PluginInitContext; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.repository.Repository; import com.linecorp.centraldogma.server.storage.repository.RepositoryManager; @@ -92,10 +91,9 @@ public final class ControlPlanePlugin extends AllReplicasPlugin { @Override public void init(PluginInitContext pluginInitContext) { - final CommandExecutor commandExecutor = pluginInitContext.commandExecutor(); - final long currentTimeMillis = System.currentTimeMillis(); - initializeInternalRepos(commandExecutor, currentTimeMillis, - ImmutableList.of(CLUSTER_REPO, ENDPOINT_REPO, LISTENER_REPO, ROUTE_REPO)); + final InternalProjectInitializer projectInitializer = pluginInitContext.internalProjectInitializer(); + projectInitializer.initializeInternalRepos( + ImmutableList.of(CLUSTER_REPO, ENDPOINT_REPO, LISTENER_REPO, ROUTE_REPO)); final ServerBuilder sb = pluginInitContext.serverBuilder(); diff --git a/xds/src/test/java/com/linecorp/centraldogma/xds/internal/XdsTestUtil.java b/xds/src/test/java/com/linecorp/centraldogma/xds/internal/XdsTestUtil.java index 4f1de908ee..d75c757a87 100644 --- a/xds/src/test/java/com/linecorp/centraldogma/xds/internal/XdsTestUtil.java +++ b/xds/src/test/java/com/linecorp/centraldogma/xds/internal/XdsTestUtil.java @@ -15,7 +15,7 @@ */ package com.linecorp.centraldogma.xds.internal; -import static com.linecorp.centraldogma.server.internal.storage.project.ProjectInitializer.INTERNAL_PROJECT_DOGMA; +import static com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer.INTERNAL_PROJECT_DOGMA; import static com.linecorp.centraldogma.xds.internal.ControlPlanePlugin.CLUSTER_FILE; import static com.linecorp.centraldogma.xds.internal.ControlPlanePlugin.CLUSTER_REPO; import static com.linecorp.centraldogma.xds.internal.ControlPlanePlugin.ENDPOINT_FILE;