Skip to content

Commit a201c54

Browse files
nmorgan-cbToshi
nmorgan-cb
and
Toshi
committed
fix(generator): isolate generated source checks
Co-authored-by: Toshi <toshi-noreply@coinbase.com>
1 parent 77f417c commit a201c54

4 files changed

Lines changed: 240 additions & 60 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,7 @@ generate:
1414
mvn -B spotless:apply
1515

1616
check-generated:
17-
$(MAKE) generate
18-
git diff --exit-code -- src/main/java tools/model-generator/generated-files.json tools/model-generator/generated-model-files.json
17+
mvn -B -f tools/model-generator/pom.xml compile exec:java@generate-models -Dexec.args="--check"
1918

2019
generate-live-diff:
2120
mvn -B -f tools/model-generator/pom.xml compile exec:java@generate-models -Dexec.args="--live-diff"

‎tools/model-generator/src/main/java/com/coinbase/tools/modelgenerator/Main.java‎

Lines changed: 118 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@
1616
package com.coinbase.tools.modelgenerator;
1717

1818
import java.io.IOException;
19+
import java.nio.charset.StandardCharsets;
1920
import java.nio.file.Files;
2021
import java.nio.file.Path;
2122
import java.util.ArrayList;
23+
import java.util.Collections;
2224
import java.util.LinkedHashMap;
2325
import java.util.List;
2426
import java.util.Map;
@@ -54,10 +56,13 @@ static void run(String[] args, GeneratorPaths paths) throws Exception {
5456
skipModels = true;
5557
}
5658

57-
List<String> changes = new ArrayList<>();
5859
if (check && !liveDiff) {
59-
changes.addAll(checkModelsInIsolation(paths, spec));
60-
} else if (!skipModels && !check) {
60+
List<String> changes = checkGeneratedInIsolation(paths, spec, configuration);
61+
reportChanges(changes);
62+
return;
63+
}
64+
65+
if (!skipModels && !check) {
6166
new OpenApiGenerator(spec.toString(), paths.rawRoot()).generateModels();
6267
new PostProcessor(
6368
paths.rawRoot(),
@@ -70,63 +75,141 @@ static void run(String[] args, GeneratorPaths paths) throws Exception {
7075
.processModels();
7176
}
7277

73-
SpecModels.Document document = SpecParser.load(spec);
74-
NamingResolver names =
75-
new NamingResolver(configuration.nameReplacements(), configuration.modelTypeMappings());
76-
List<OperationBinding> bindings = OperationBindingGenerator.deriveAll(document, configuration);
77-
JavaTypeResolver types = new JavaTypeResolver(document, names, configuration.sharedModelMappings());
78-
Map<Path, String> sources = new LinkedHashMap<>();
79-
sources.putAll(RequestPhase.render(document, bindings, types, names));
80-
sources.putAll(ResponsePhase.render(document, bindings, types, names));
81-
sources.putAll(ServicePhase.render(document, bindings, configuration, names));
82-
sources.putAll(FactoryPhase.render(bindings));
83-
changes.addAll(
78+
Map<Path, String> sources = renderClientSources(spec, configuration);
79+
List<String> changes =
8480
GeneratedSourceReconciler.diff(
85-
paths.sourceRoot(), sources, configuration.protectedFiles(), paths.manifest()));
81+
paths.sourceRoot(), sources, configuration.protectedFiles(), paths.manifest());
8682

8783
if (check) {
88-
for (String change : changes) {
89-
System.out.println(change);
90-
}
91-
if (!changes.isEmpty()) {
92-
throw new IllegalStateException("Generated source is out of date (" + changes.size() + " changes)");
93-
}
84+
reportChanges(changes);
9485
} else {
9586
GeneratedSourceReconciler.write(
9687
paths.sourceRoot(), sources, configuration.protectedFiles(), paths.manifest());
9788
System.out.println("Generated " + sources.size() + " client-surface files");
9889
}
9990
}
10091

101-
/** Renders models into a temporary source tree so checks never touch committed SDK files. */
102-
static List<String> checkModelsInIsolation(GeneratorPaths paths, Path spec) throws Exception {
103-
Files.createDirectories(paths.rawRoot());
104-
Path stagingRoot = Files.createTempDirectory(paths.rawRoot(), "check-models-");
92+
/**
93+
* Runs the complete write-mode pipeline against a disposable copy of the source tree.
94+
*
95+
* <p>The staged project receives the same Spotless normalization as {@code make generate}; only its
96+
* rendered, manifest-owned files are compared with the repository. The repository itself is never
97+
* used as an output directory in check mode.
98+
*/
99+
static List<String> checkGeneratedInIsolation(
100+
GeneratorPaths paths, Path spec, GeneratorConfiguration configuration) throws Exception {
101+
Path stagingRoot = Files.createTempDirectory("prime-sdk-java-generator-check-");
105102
try {
106-
Path rawGenerationRoot = stagingRoot.resolve("raw-generation");
107103
Path stagedSourceRoot = stagingRoot.resolve("src/main/java");
108104
Path stagedModelRoot = stagedSourceRoot.resolve("com/coinbase/prime/model");
109-
Path stagedManifest = stagingRoot.resolve("generated-model-files.json");
110-
new OpenApiGenerator(spec.toString(), rawGenerationRoot).generateModels();
105+
Path stagedModelManifest = stagingRoot.resolve("tools/model-generator/generated-model-files.json");
106+
Path stagedClientManifest = stagingRoot.resolve("tools/model-generator/generated-files.json");
107+
108+
copyProjectInputs(paths, stagingRoot, stagedSourceRoot, stagedModelManifest, stagedClientManifest);
109+
Path stagedRawRoot = stagingRoot.resolve("generated");
110+
new OpenApiGenerator(spec.toString(), stagedRawRoot, paths.root()).generateModels();
111111
new PostProcessor(
112-
rawGenerationRoot,
112+
stagedRawRoot,
113113
stagedSourceRoot,
114114
stagedModelRoot,
115115
stagedModelRoot.resolve("enums"),
116116
stagedModelRoot.resolve("errors"),
117117
spec,
118-
stagedManifest)
118+
stagedModelManifest)
119119
.processModels();
120-
return GeneratedSourceReconciler.diff(
121-
paths.sourceRoot(),
122-
GeneratedSourceReconciler.readOwnedSources(stagedSourceRoot, stagedManifest),
123-
java.util.Collections.emptySet(),
124-
paths.modelManifest());
120+
GeneratedSourceReconciler.write(
121+
stagedSourceRoot,
122+
renderClientSources(spec, configuration),
123+
configuration.protectedFiles(),
124+
stagedClientManifest);
125+
formatStagedSources(stagingRoot);
126+
127+
List<String> changes = new ArrayList<>();
128+
changes.addAll(
129+
GeneratedSourceReconciler.diff(
130+
paths.sourceRoot(),
131+
GeneratedSourceReconciler.readOwnedSources(stagedSourceRoot, stagedModelManifest),
132+
Collections.emptySet(),
133+
paths.modelManifest()));
134+
changes.addAll(
135+
GeneratedSourceReconciler.diff(
136+
paths.sourceRoot(),
137+
GeneratedSourceReconciler.readOwnedSources(stagedSourceRoot, stagedClientManifest),
138+
configuration.protectedFiles(),
139+
paths.manifest()));
140+
addManifestChangeIfPresent(changes, paths.modelManifest(), stagedModelManifest);
141+
addManifestChangeIfPresent(changes, paths.manifest(), stagedClientManifest);
142+
Collections.sort(changes);
143+
return Collections.unmodifiableList(changes);
125144
} finally {
126145
FileUtils.deleteDirectory(stagingRoot.toFile());
127146
}
128147
}
129148

149+
private static void copyProjectInputs(
150+
GeneratorPaths paths,
151+
Path stagingRoot,
152+
Path stagedSourceRoot,
153+
Path stagedModelManifest,
154+
Path stagedClientManifest)
155+
throws IOException {
156+
Files.copy(paths.root().resolve("pom.xml"), stagingRoot.resolve("pom.xml"));
157+
FileUtils.copyDirectory(paths.sourceRoot().toFile(), stagedSourceRoot.toFile());
158+
copyIfPresent(paths.modelManifest(), stagedModelManifest);
159+
copyIfPresent(paths.manifest(), stagedClientManifest);
160+
}
161+
162+
private static void copyIfPresent(Path source, Path target) throws IOException {
163+
if (Files.exists(source)) {
164+
Files.createDirectories(target.getParent());
165+
Files.copy(source, target);
166+
}
167+
}
168+
169+
private static void formatStagedSources(Path stagingRoot) throws IOException, InterruptedException {
170+
Process process =
171+
new ProcessBuilder(
172+
"mvn", "-B", "-f", stagingRoot.resolve("pom.xml").toString(), "spotless:apply")
173+
.redirectErrorStream(true)
174+
.start();
175+
String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8);
176+
if (process.waitFor() != 0) {
177+
throw new IOException("Could not format staged generated sources:\n" + output);
178+
}
179+
}
180+
181+
private static void addManifestChangeIfPresent(
182+
List<String> changes, Path committedManifest, Path stagedManifest) throws IOException {
183+
if (Files.exists(committedManifest)
184+
&& !Files.readString(committedManifest).equals(Files.readString(stagedManifest))) {
185+
changes.add("CHANGE " + committedManifest.getFileName());
186+
}
187+
}
188+
189+
private static Map<Path, String> renderClientSources(
190+
Path spec, GeneratorConfiguration configuration) throws IOException {
191+
SpecModels.Document document = SpecParser.load(spec);
192+
NamingResolver names =
193+
new NamingResolver(configuration.nameReplacements(), configuration.modelTypeMappings());
194+
List<OperationBinding> bindings = OperationBindingGenerator.deriveAll(document, configuration);
195+
JavaTypeResolver types = new JavaTypeResolver(document, names, configuration.sharedModelMappings());
196+
Map<Path, String> sources = new LinkedHashMap<>();
197+
sources.putAll(RequestPhase.render(document, bindings, types, names));
198+
sources.putAll(ResponsePhase.render(document, bindings, types, names));
199+
sources.putAll(ServicePhase.render(document, bindings, configuration, names));
200+
sources.putAll(FactoryPhase.render(bindings));
201+
return sources;
202+
}
203+
204+
private static void reportChanges(List<String> changes) {
205+
for (String change : changes) {
206+
System.out.println(change);
207+
}
208+
if (!changes.isEmpty()) {
209+
throw new IllegalStateException("Generated source is out of date (" + changes.size() + " changes)");
210+
}
211+
}
212+
130213
private static boolean has(String[] args, String value) {
131214
for (String arg : args) {
132215
if (value.equals(arg)) {

‎tools/model-generator/src/main/java/com/coinbase/tools/modelgenerator/OpenApiGenerator.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,18 @@ public class OpenApiGenerator {
3434

3535
private final String specLocation;
3636
private final Path outputDir;
37+
private final Path projectRoot;
3738
private final ObjectMapper objectMapper = new ObjectMapper();
3839

3940
public OpenApiGenerator(String specLocation, Path outputDir) {
41+
this(specLocation, outputDir, null);
42+
}
43+
44+
/** Allows isolated checks to write outside the repository while still using repository templates. */
45+
public OpenApiGenerator(String specLocation, Path outputDir, Path projectRoot) {
4046
this.specLocation = specLocation;
4147
this.outputDir = outputDir;
48+
this.projectRoot = projectRoot;
4249
}
4350

4451
public void generateModels() throws IOException {
@@ -141,6 +148,9 @@ public void generateModels() throws IOException {
141148
}
142149

143150
private Path findProjectRoot() {
151+
if (projectRoot != null) {
152+
return projectRoot;
153+
}
144154
Path current = outputDir.getParent();
145155
while (current != null) {
146156
if (current.resolve("pom.xml").toFile().exists() &&

‎tools/model-generator/src/test/java/com/coinbase/tools/modelgenerator/MainCheckTest.java‎

Lines changed: 111 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -15,38 +15,126 @@
1515
*/
1616
package com.coinbase.tools.modelgenerator;
1717

18+
import static org.junit.jupiter.api.Assertions.assertEquals;
1819
import static org.junit.jupiter.api.Assertions.assertFalse;
20+
import static org.junit.jupiter.api.Assertions.assertThrows;
1921
import static org.junit.jupiter.api.Assertions.assertTrue;
2022

2123
import java.nio.file.Files;
2224
import java.nio.file.Path;
25+
import java.security.MessageDigest;
26+
import java.util.LinkedHashMap;
2327
import java.util.List;
28+
import java.util.Map;
29+
import java.util.stream.Collectors;
30+
import java.util.stream.Stream;
31+
import org.apache.commons.io.FileUtils;
2432
import org.junit.jupiter.api.Test;
2533

2634
class MainCheckTest {
2735
@Test
28-
void checkRendersModelsInIsolationAndReportsDriftWithoutWritingTheSdkTree() throws Exception {
29-
Path root = Files.createTempDirectory("isolated-model-check");
30-
Path sourceRoot = root.resolve("src/main/java");
31-
Path spec = root.resolve("apiSpec/openapi.yaml");
32-
Files.createDirectories(sourceRoot.resolve("com/coinbase/prime"));
33-
Files.createDirectories(spec.getParent());
34-
Files.writeString(root.resolve("pom.xml"), "<project/>\n");
35-
Files.writeString(spec, String.join("\n",
36-
"openapi: 3.0.0",
37-
"info: { title: test, version: 1.0.0 }",
38-
"paths: {}",
39-
"components:",
40-
" schemas:",
41-
" Thing:",
42-
" type: object",
43-
" properties:",
44-
" id: { type: string }",
45-
""));
46-
47-
List<String> changes = Main.checkModelsInIsolation(GeneratorPaths.forRoot(root), spec);
48-
49-
assertTrue(changes.stream().anyMatch(change -> change.endsWith("com/coinbase/prime/model/Thing.java")));
50-
assertFalse(Files.exists(sourceRoot.resolve("com/coinbase/prime/model/Thing.java")));
36+
void checkCliLeavesFixtureSourcesAndManifestStateByteForByteUnchanged() throws Exception {
37+
Path root = Files.createTempDirectory("isolated-generator-check");
38+
try {
39+
writeFixture(root);
40+
Map<String, String> before = snapshot(root);
41+
42+
IllegalStateException exception =
43+
assertThrows(
44+
IllegalStateException.class,
45+
() -> {
46+
Main.run(new String[] {"--check"}, GeneratorPaths.forRoot(root));
47+
});
48+
49+
assertTrue(exception.getMessage().contains("out of date"), exception.getMessage());
50+
assertEquals(before, snapshot(root));
51+
assertFalse(Files.exists(root.resolve("generated")));
52+
assertFalse(Files.exists(root.resolve("tools/model-generator/generated-files.json")));
53+
assertFalse(Files.exists(root.resolve("tools/model-generator/generated-model-files.json")));
54+
assertFalse(Files.exists(root.resolve("src/main/java/com/coinbase/prime/things/ListThingsResponse.java")));
55+
assertFalse(Files.exists(root.resolve("src/main/java/com/coinbase/prime/model/Thing.java")));
56+
} finally {
57+
FileUtils.deleteDirectory(root.toFile());
58+
}
59+
}
60+
61+
private static void writeFixture(Path root) throws Exception {
62+
Path sourceRoot = root.resolve("src/main/java/com/coinbase/prime");
63+
Files.createDirectories(sourceRoot);
64+
Files.createDirectories(root.resolve("apiSpec"));
65+
Files.createDirectories(root.resolve("tools/model-generator/config"));
66+
Files.writeString(root.resolve("pom.xml"), fixturePom());
67+
Files.writeString(sourceRoot.resolve("Existing.java"), "package com.coinbase.prime;\nclass Existing {}\n");
68+
Files.writeString(
69+
root.resolve("tools/model-generator/config/generator-config.json"),
70+
"{\"specUrl\":\"unused\",\"committedSpecPath\":\"apiSpec/openapi.yaml\"}\n");
71+
Files.writeString(root.resolve("tools/model-generator/config/operations-overrides.json"), "[]\n");
72+
Files.writeString(
73+
root.resolve("apiSpec/openapi.yaml"),
74+
String.join(
75+
"\n",
76+
"openapi: 3.0.0",
77+
"info: { title: test, version: 1.0.0 }",
78+
"paths:",
79+
" /v1/things:",
80+
" get:",
81+
" operationId: PrimeRESTAPI_ListThings",
82+
" tags: [Things]",
83+
" responses:",
84+
" '200':",
85+
" description: Success",
86+
" content:",
87+
" application/json:",
88+
" schema:",
89+
" type: object",
90+
" properties:",
91+
" thing: { $ref: '#/components/schemas/Thing' }",
92+
"components:",
93+
" schemas:",
94+
" Thing:",
95+
" type: object",
96+
" properties:",
97+
" id: { type: string }",
98+
""));
99+
}
100+
101+
private static String fixturePom() {
102+
return String.join(
103+
"\n",
104+
"<project xmlns=\"http://maven.apache.org/POM/4.0.0\">",
105+
" <modelVersion>4.0.0</modelVersion>",
106+
" <groupId>test</groupId>",
107+
" <artifactId>fixture</artifactId>",
108+
" <version>1.0.0</version>",
109+
" <build><plugins><plugin>",
110+
" <groupId>com.diffplug.spotless</groupId>",
111+
" <artifactId>spotless-maven-plugin</artifactId>",
112+
" <version>2.43.0</version>",
113+
" <configuration><java><googleJavaFormat>",
114+
" <version>1.24.0</version><style>GOOGLE</style>",
115+
" </googleJavaFormat></java></configuration>",
116+
" </plugin></plugins></build>",
117+
"</project>",
118+
"");
119+
}
120+
121+
private static Map<String, String> snapshot(Path root) throws Exception {
122+
Map<String, String> files = new LinkedHashMap<>();
123+
try (Stream<Path> paths = Files.walk(root)) {
124+
List<Path> regularFiles = paths.filter(Files::isRegularFile).sorted().collect(Collectors.toList());
125+
for (Path file : regularFiles) {
126+
files.put(root.relativize(file).toString(), sha256(Files.readAllBytes(file)));
127+
}
128+
}
129+
return files;
130+
}
131+
132+
private static String sha256(byte[] content) throws Exception {
133+
byte[] digest = MessageDigest.getInstance("SHA-256").digest(content);
134+
StringBuilder hex = new StringBuilder();
135+
for (byte value : digest) {
136+
hex.append(String.format("%02x", value));
137+
}
138+
return hex.toString();
51139
}
52140
}

0 commit comments

Comments
 (0)