Skip to content

Validate MODULE_DEPENDENCIES using tracing from Clang #635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions Sources/SWBCore/Dependencies.swift
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,48 @@ public struct ModuleDependenciesContext: Sendable, SerializableCodable {
self.init(validate: validate, moduleDependencies: settings.moduleDependencies, fixItContext: fixItContext)
}

// For Clang imports, we do not get the import locations and do not know whether it was a public
// import (which depends on whether the import is in an installed header file).
public func makeDiagnostics(files: [Path]) -> [Diagnostic] {
guard validate != .no else { return [] }

// The following is a provisional/incomplete mechanism for resolving a module dependency from a file path.
// For now, just grab the framework name and assume there is a module with the same name.
func findFrameworkName(_ file: Path) -> String? {
if file.fileExtension == "framework" {
return file.basenameWithoutSuffix
}
return file.dirname.isEmpty || file.dirname.isRoot ? nil : findFrameworkName(file.dirname)
}

let moduleDependencyNames = moduleDependencies.map { $0.name }
let fileNames = files.compactMap { findFrameworkName($0) }
let missingDeps = fileNames.filter {
return !moduleDependencyNames.contains($0)
}.map {
ModuleDependency(name: $0, accessLevel: .Private)
}

guard !missingDeps.isEmpty else { return [] }

let behavior: Diagnostic.Behavior = validate == .yesError ? .error : .warning

let fixIt = fixItContext?.makeFixIt(newModules: missingDeps)
let fixIts = fixIt.map { [$0] } ?? []

let message = "Missing entries in \(BuiltinMacros.MODULE_DEPENDENCIES.name): \(missingDeps.map { $0.asBuildSettingEntryQuotedIfNeeded }.sorted().joined(separator: " "))"

let location: Diagnostic.Location = fixIt.map {
Diagnostic.Location.path($0.sourceRange.path, line: $0.sourceRange.endLine, column: $0.sourceRange.endColumn)
} ?? Diagnostic.Location.buildSetting(BuiltinMacros.MODULE_DEPENDENCIES)

return [Diagnostic(
behavior: behavior,
location: location,
data: DiagnosticData(message),
fixIts: fixIts)]
}

/// Nil `imports` means the current toolchain doesn't have the features to gather imports. This is temporarily required to support running against older toolchains.
public func makeDiagnostics(imports: [(ModuleDependency, importLocations: [Diagnostic.Location])]?) -> [Diagnostic] {
guard validate != .no else { return [] }
Expand Down Expand Up @@ -181,4 +223,9 @@ public struct ModuleDependenciesContext: Sendable, SerializableCodable {
return Diagnostic.FixIt(sourceRange: sourceRange, newText: newText)
}
}

func signatureData() -> String {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this quite belongs here. For one, the type is serializable/codable, so clients can serialize it to get a signature blob. And two, what actually goes into a signature depends on each specific client task, so there can't be a central definition like this. That said, I think the clang task signature actually should contain the full value of ModuleDependenciesContext (and probably the entire task Payload tbh? @owenv is that not already the case?) instead of just validate+names, because things like access level and fixit context do influence the diagnostics the clang task emits, so we'd want to re-run when the user changes any of those to avoid ending up with stale diagnostics that give you incorrect fixits.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably not the entire payload because it's fairly large and a lot of the content doesn't need to be included because it's redundant with the command line. But yeah, I would just serialize ModuleDependenciesContext to form a signature for it

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops, that means that my Swift change is incorrect for incremental builds, since it doesn't add this context to its signature. Going to fix that.

let moduleNames = moduleDependencies.map { $0.name }
return "validate:\(validate),modules:\(moduleNames.joined(separator: ":"))"
}
}
39 changes: 35 additions & 4 deletions Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,10 @@ public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEd

public let fileNameMapPath: Path?

fileprivate init(serializedDiagnosticsPath: Path?, indexingPayload: ClangIndexingPayload?, explicitModulesPayload: ClangExplicitModulesPayload? = nil, outputObjectFilePath: Path? = nil, fileNameMapPath: Path? = nil, developerPathString: String? = nil) {
public let moduleDependenciesContext: ModuleDependenciesContext?
public let traceFile: Path?
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should have the Path suffix like fileNameMapPath


fileprivate init(serializedDiagnosticsPath: Path?, indexingPayload: ClangIndexingPayload?, explicitModulesPayload: ClangExplicitModulesPayload? = nil, outputObjectFilePath: Path? = nil, fileNameMapPath: Path? = nil, developerPathString: String? = nil, moduleDependenciesContext: ModuleDependenciesContext? = nil, traceFile: Path? = nil) {
if let developerPathString, explicitModulesPayload == nil {
self.dependencyInfoEditPayload = .init(removablePaths: [], removableBasenames: [], developerPath: Path(developerPathString))
} else {
Expand All @@ -443,27 +446,33 @@ public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEd
self.explicitModulesPayload = explicitModulesPayload
self.outputObjectFilePath = outputObjectFilePath
self.fileNameMapPath = fileNameMapPath
self.moduleDependenciesContext = moduleDependenciesContext
self.traceFile = traceFile
}

public func serialize<T: Serializer>(to serializer: T) {
serializer.serializeAggregate(6) {
serializer.serializeAggregate(8) {
serializer.serialize(serializedDiagnosticsPath)
serializer.serialize(indexingPayload)
serializer.serialize(explicitModulesPayload)
serializer.serialize(outputObjectFilePath)
serializer.serialize(fileNameMapPath)
serializer.serialize(dependencyInfoEditPayload)
serializer.serialize(moduleDependenciesContext)
serializer.serialize(traceFile)
}
}

public init(from deserializer: any Deserializer) throws {
try deserializer.beginAggregate(6)
try deserializer.beginAggregate(8)
self.serializedDiagnosticsPath = try deserializer.deserialize()
self.indexingPayload = try deserializer.deserialize()
self.explicitModulesPayload = try deserializer.deserialize()
self.outputObjectFilePath = try deserializer.deserialize()
self.fileNameMapPath = try deserializer.deserialize()
self.dependencyInfoEditPayload = try deserializer.deserialize()
self.moduleDependenciesContext = try deserializer.deserialize()
self.traceFile = try deserializer.deserialize()
}
}

Expand Down Expand Up @@ -1156,6 +1165,22 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
dependencyData = nil
}

let moduleDependenciesContext = cbc.producer.moduleDependenciesContext
let traceFile: Path?
if clangInfo?.hasFeature("print-headers-direct-per-file") ?? false,
(moduleDependenciesContext?.validate ?? .defaultValue) != .no {
let file = Path(outputNode.path.str + ".trace.json")
commandLine += [
"-Xclang", "-header-include-file",
"-Xclang", file.str,
"-Xclang", "-header-include-filtering=direct-per-file",
"-Xclang", "-header-include-format=json"
]
traceFile = file
} else {
traceFile = nil
}

// Add the diagnostics serialization flag. We currently place the diagnostics file right next to the output object file.
let diagFilePath: Path?
if let serializedDiagnosticsOptions = self.serializedDiagnosticsOptions(scope: cbc.scope, outputPath: outputNode.path) {
Expand Down Expand Up @@ -1266,7 +1291,9 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
explicitModulesPayload: explicitModulesPayload,
outputObjectFilePath: shouldGenerateRemarks ? outputNode.path : nil,
fileNameMapPath: verifierPayload?.fileNameMapPath,
developerPathString: recordSystemHeaderDepsOutsideSysroot ? cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR).str : nil
developerPathString: recordSystemHeaderDepsOutsideSysroot ? cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR).str : nil,
moduleDependenciesContext: moduleDependenciesContext,
traceFile: traceFile
)

var inputNodes: [any PlannedNode] = inputDeps.map { delegate.createNode($0) }
Expand Down Expand Up @@ -1316,6 +1343,10 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
extraInputs = []
}

if let moduleDependenciesContext {
additionalSignatureData += moduleDependenciesContext.signatureData()
}

// Finally, create the task.
delegate.createTask(type: self, dependencyData: dependencyData, payload: payload, ruleInfo: ruleInfo, additionalSignatureData: additionalSignatureData, commandLine: commandLine, additionalOutput: additionalOutput, environment: environmentBindings, workingDirectory: compilerWorkingDirectory(cbc), inputs: inputNodes + extraInputs, outputs: [outputNode], action: action ?? delegate.taskActionCreationDelegate.createDeferredExecutionTaskActionIfRequested(userPreferences: cbc.producer.userPreferences), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing, additionalTaskOrderingOptions: [.compilationForIndexableSourceFile], usesExecutionInputs: usesExecutionInputs, showEnvironment: true, priority: .preferred)

Expand Down
5 changes: 5 additions & 0 deletions Sources/SWBCore/ToolInfo/ClangToolInfo.swift
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public struct DiscoveredClangToolSpecInfo: DiscoveredCommandLineToolSpecInfo {
case wSystemHeadersInModule = "Wsystem-headers-in-module"
case extractAPISupportsCPlusPlus = "extract-api-supports-cpp"
case deploymentTargetEnvironmentVariables = "deployment-target-environment-variables"
case printHeadersDirectPerFile = "print-headers-direct-per-file"
}
public var toolFeatures: ToolFeatures<FeatureFlag>
public func hasFeature(_ feature: String) -> Bool {
Expand All @@ -46,6 +47,10 @@ public struct DiscoveredClangToolSpecInfo: DiscoveredCommandLineToolSpecInfo {
if feature == FeatureFlag.extractAPISupportsCPlusPlus.rawValue {
return clangVersion > Version(17)
}
// FIXME: Remove once the feature flag is added to clang.
if feature == FeatureFlag.printHeadersDirectPerFile.rawValue, let clangVersion {
return clangVersion >= Version(1700, 3, 10, 2)
}
return toolFeatures.has(feature)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,20 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA
casDBs = nil
}

// Check if verifying dependencies from trace data is enabled.
var traceFile: Path? = nil
var moduleDependenciesContext: ModuleDependenciesContext? = nil
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit but typically we'd make these two let and have else set them to nil, just like what you did with let traceFile: Path? in CCompiler.swift

if let payload = task.payload as? ClangTaskPayload {
traceFile = payload.traceFile
moduleDependenciesContext = payload.moduleDependenciesContext
}
if let traceFile {
// Remove the trace output file if it already exists.
if executionDelegate.fs.exists(traceFile) {
try executionDelegate.fs.remove(traceFile)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does clang create or append to this particular trace file just like the ld trace we looked at?

}
}

var lastResult: CommandResult? = nil
for command in dependencyInfo.commands {
if let casDBs {
Expand Down Expand Up @@ -304,6 +318,24 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA
return lastResult ?? .failed
}
}

if let moduleDependenciesContext, let traceFile, lastResult == .succeeded {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If clang doesn't have the feature, traceFile will be nil here, so we'll skip checking altogether, even if the user requested VALIDATE_MODULE_DEPENDENCIES. The Swift version on the other hand emits an error if VALIDATE_MODULE_DEPENDENCIES is set but the toolchain doesn't have the required support, mainly so that the test can check for that when it's running against older toolchains.

// Verify the dependencies from the trace data.
let fs = executionDelegate.fs
let traceData = try JSONDecoder().decode(Array<TraceData>.self, from: fs.readMemoryMapped(traceFile))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why use readMemoryMapped here? I think we don't have a lot of clarity around when to use read vs readMemoryMapped and should probably default to read.


var allFiles = Set<Path>()
traceData.forEach { allFiles.formUnion(Set($0.includes)) }
let diagnostics = moduleDependenciesContext.makeDiagnostics(files: Array(allFiles))
for diagnostic in diagnostics {
outputDelegate.emit(diagnostic)
}

if diagnostics.contains(where: { $0.behavior == .error }) {
return .failed
}
}

return lastResult ?? .failed
} catch {
outputDelegate.emitError("\(error)")
Expand Down Expand Up @@ -431,3 +463,10 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA
)
}
}

// Results from tracing header includes with "direct-per-file" filtering.
// This is used to validate dependencies.
fileprivate struct TraceData: Decodable {
let source: Path
let includes: [Path]
}
4 changes: 4 additions & 0 deletions Sources/SWBTestSupport/CoreBasedTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,10 @@ extension CoreBasedTests {
if clangInfo.clangVersion > Version(17) {
realToolFeatures.insert(.extractAPISupportsCPlusPlus)
}
if let clangVersion = clangInfo.clangVersion, clangVersion >= Version(1700, 3, 10, 2) {
realToolFeatures.insert(.printHeadersDirectPerFile)
}

return ToolFeatures(realToolFeatures)
}
}
Expand Down
71 changes: 70 additions & 1 deletion Tests/SWBBuildSystemTests/DependencyValidationTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ fileprivate struct DependencyValidationTests: CoreBasedTests {
}

@Test(.requireSDKs(.host))
func validateModuleDependencies() async throws {
func validateModuleDependenciesSwift() async throws {
try await withTemporaryDirectory { tmpDir in
let testWorkspace = try await TestWorkspace(
"Test",
Expand Down Expand Up @@ -458,4 +458,73 @@ fileprivate struct DependencyValidationTests: CoreBasedTests {
}
}
}

@Test(.requireSDKs(.host))
func validateModuleDependenciesClang() async throws {
try await withTemporaryDirectory { tmpDir async throws -> Void in
let testWorkspace = TestWorkspace(
"Test",
sourceRoot: tmpDir.join("Test"),
projects: [
TestProject(
"aProject",
groupTree: TestGroup(
"Sources", path: "Sources",
children: [
TestFile("CoreFoo.m")
]),
buildConfigurations: [
TestBuildConfiguration(
"Debug",
buildSettings: [
"PRODUCT_NAME": "$(TARGET_NAME)",
"CLANG_ENABLE_MODULES": "YES",
"CLANG_ENABLE_EXPLICIT_MODULES": "YES",
"GENERATE_INFOPLIST_FILE": "YES",
"MODULE_DEPENDENCIES": "Foundation",
"VALIDATE_MODULE_DEPENDENCIES": "YES_ERROR",
"SDKROOT": "$(HOST_PLATFORM)",
"SUPPORTED_PLATFORMS": "$(HOST_PLATFORM)",
"DSTROOT": tmpDir.join("dstroot").str,
]
)
],
targets: [
TestStandardTarget(
"CoreFoo", type: .framework,
buildPhases: [
TestSourcesBuildPhase(["CoreFoo.m"]),
TestFrameworksBuildPhase()
])
])
]
)

let tester = try await BuildOperationTester(getCore(), testWorkspace, simulated: false)
let SRCROOT = testWorkspace.sourceRoot.join("aProject")

// Write the source files.
try await tester.fs.writeFileContents(SRCROOT.join("Sources/CoreFoo.m")) { contents in
contents <<< """
#include <Foundation/Foundation.h>
#include <Accelerate/Accelerate.h>

void f0(void) { };
"""
}

if try await clangFeatures.has(.printHeadersDirectPerFile) {
// Expect complaint about undeclared dependency
try await tester.checkBuild(parameters: BuildParameters(configuration: "Debug"), runDestination: .host, persistent: true) { results in
results.checkError(.contains("Missing entries in MODULE_DEPENDENCIES: Accelerate"))
}

// Declaring dependencies resolves the problem
try await tester.checkBuild(parameters: BuildParameters(configuration: "Debug", overrides: ["MODULE_DEPENDENCIES": "Foundation Accelerate"]), runDestination: .host, persistent: true) { results in
results.checkNoErrors()
}
}
}
}

}
Loading
Loading