Skip to content

[XABT] Move marshal method generation to a "linker step". #10027

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

[XABT] Move marshal method generation to a "linker step". #10027

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 102 additions & 0 deletions src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/RewriteMarshalMethodsStep.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using Java.Interop.Tools.JavaCallableWrappers;
using Microsoft.Android.Build.Tasks;
using Microsoft.Build.Utilities;
using Mono.Cecil;
using Mono.Linker;
using Mono.Linker.Steps;
using Xamarin.Android.Tasks;
using Xamarin.Android.Tools;

namespace MonoDroid.Tuner;

/// <summary>
/// Scans an assembly for marshal methods and converts them to LLVM marshal methods.
/// </summary>
public class RewriteMarshalMethodsStep : BaseStep, IAssemblyModifierPipelineStep
{
public TaskLoggingHelper Log { get; set; }

bool? brokenExceptionTransitionsEnabled;

public RewriteMarshalMethodsStep (TaskLoggingHelper log)
{
Log = log;
}

public void ProcessAssembly (AssemblyDefinition assembly, StepContext context)
{
if (!context.IsAndroidAssembly)
return;

var action = Annotations.HasAction (assembly) ? Annotations.GetAction (assembly) : AssemblyAction.Skip;

if (action == AssemblyAction.Delete)
return;

// We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed
// in order to properly generate wrapper methods in the marshal methods assembly rewriter.
// We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for.
if (!brokenExceptionTransitionsEnabled.HasValue) {
var environmentParser = new EnvironmentFilesParser ();
brokenExceptionTransitionsEnabled = environmentParser.AreBrokenExceptionTransitionsEnabled (context.Environments);
}

var collection = MarshalMethodsCollection.FromAssembly (context.Architecture, assembly, Context.Resolver, Log);

var state = new NativeCodeGenState (context.Architecture, Context, Context.Resolver, [], [], collection);
Run (state, context);

var stateObject = MarshalMethodCecilAdapter.CreateNativeCodeGenState (context.Architecture, state);
var destinationMarshalMethodsXml = MarshalMethodsXmlFile.GetMarshalMethodsXmlFilePath (context.Destination.ItemSpec);

MarshalMethodsXmlFile.Export (destinationMarshalMethodsXml, context.Architecture, stateObject, Log);

// TODO: Only return true if we actually modified the assembly
context.IsAssemblyModified = true;
}

void Run (NativeCodeGenState state, StepContext context)
{
if (state.Classifier is null) {
Log.LogError ("state.Classifier cannot be null if marshal methods are enabled");
return;
}

if (!context.EnableManagedMarshalMethodsLookup) {
RewriteMethods (state, brokenExceptionTransitionsEnabled.GetValueOrDefault ());
state.Classifier.AddSpecialCaseMethods ();
} else {
// We need to run `AddSpecialCaseMethods` before `RewriteMarshalMethods` so that we can see the special case
// methods (such as TypeManager.n_Activate_mm) when generating the managed lookup tables.
state.Classifier.AddSpecialCaseMethods ();
state.ManagedMarshalMethodsLookupInfo = new ManagedMarshalMethodsLookupInfo (Log);
RewriteMethods (state, brokenExceptionTransitionsEnabled.GetValueOrDefault ());
}

Log.LogDebugMessage ($"[{state.TargetArch}] Number of generated marshal methods: {state.Classifier.MarshalMethods.Count}");
if (state.Classifier.DynamicallyRegisteredMarshalMethods.Count > 0) {
Log.LogWarning ($"[{state.TargetArch}] Number of methods in the project that will be registered dynamically: {state.Classifier.DynamicallyRegisteredMarshalMethods.Count}");
}

var wrappedCount = state.Classifier.MarshalMethods.Sum (m => m.Value.Count (m2 => m2.NeedsBlittableWorkaround));

if (wrappedCount > 0) {
// TODO: change to LogWarning once the generator can output code which requires no non-blittable wrappers
Log.LogDebugMessage ($"[{state.TargetArch}] Number of methods in the project that need marshal method wrappers: {wrappedCount}");
}
}

void RewriteMethods (NativeCodeGenState state, bool brokenExceptionTransitionsEnabled)
{
if (state.Classifier == null) {
return;
}

var rewriter = new MarshalMethodsAssemblyRewriter (Log, state.TargetArch, state.Classifier, state.Resolver, state.ManagedMarshalMethodsLookupInfo);
rewriter.Rewrite (brokenExceptionTransitionsEnabled);
}
}
28 changes: 24 additions & 4 deletions src/Xamarin.Android.Build.Tasks/Tasks/AssemblyModifierPipeline.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ public class AssemblyModifierPipeline : AndroidTask
{
public override string TaskPrefix => "AMP";

public string AndroidSdkPlatform { get; set; } = "";

public string ApplicationJavaClass { get; set; } = "";

public string CodeGenerationTarget { get; set; } = "";
Expand All @@ -37,8 +39,15 @@ public class AssemblyModifierPipeline : AndroidTask

public bool EnableMarshalMethods { get; set; }

public bool EnableManagedMarshalMethodsLookup { get; set; }

public ITaskItem [] Environments { get; set; } = [];

public bool ErrorOnCustomJavaObject { get; set; }

// If we're using ILLink, this process modifies the linked assemblies in place
protected virtual bool ModifiesAssembliesInPlace => true;

public string? PackageNamingPolicy { get; set; }

/// <summary>
Expand All @@ -64,7 +73,7 @@ public class AssemblyModifierPipeline : AndroidTask
[Required]
public string TargetName { get; set; } = "";

protected JavaPeerStyle codeGenerationTarget;
JavaPeerStyle codeGenerationTarget;

public override bool RunTask ()
{
Expand All @@ -76,6 +85,8 @@ public override bool RunTask ()

var readerParameters = new ReaderParameters {
ReadSymbols = ReadSymbols,
ReadWrite = ModifiesAssembliesInPlace || EnableMarshalMethods,
InMemory = ModifiesAssembliesInPlace || EnableMarshalMethods,
};

Dictionary<AndroidTargetArch, Dictionary<string, ITaskItem>> perArchAssemblies = MonoAndroidHelper.GetPerArchAssemblies (ResolvedAssemblies, Array.Empty<string> (), validate: false);
Expand Down Expand Up @@ -118,7 +129,7 @@ public override bool RunTask ()

Directory.CreateDirectory (Path.GetDirectoryName (destination.ItemSpec));

RunPipeline (pipeline!, source, destination);
RunPipeline (pipeline!, source, destination, perArchAssemblies [sourceArch].Values.ToArray ());
}

pipeline?.Dispose ();
Expand All @@ -138,6 +149,13 @@ protected virtual void BuildPipeline (AssemblyPipeline pipeline, MSBuildLinkCont
findJavaObjectsStep.Initialize (context);
pipeline.Steps.Add (findJavaObjectsStep);

// RewriteMarshalMethodsStep
if (EnableMarshalMethods && !Debug) {
var rewriteMarshalMethodsStep = new RewriteMarshalMethodsStep (Log);
rewriteMarshalMethodsStep.Initialize (context);
pipeline.Steps.Add (rewriteMarshalMethodsStep);
}

// SaveChangedAssemblyStep
var writerParameters = new WriterParameters {
DeterministicMvid = Deterministic,
Expand All @@ -156,13 +174,15 @@ protected virtual void BuildPipeline (AssemblyPipeline pipeline, MSBuildLinkCont
pipeline.Steps.Add (findTypeMapObjectsStep);
}

void RunPipeline (AssemblyPipeline pipeline, ITaskItem source, ITaskItem destination)
void RunPipeline (AssemblyPipeline pipeline, ITaskItem source, ITaskItem destination, ITaskItem [] archAssemblies)
{
var assembly = pipeline.Resolver.GetAssembly (source.ItemSpec);

var context = new StepContext (source, destination) {
var context = new StepContext (source, destination, AndroidSdkPlatform, Environments, archAssemblies) {
Architecture = MonoAndroidHelper.GetRequiredValidArchitecture (source),
CodeGenerationTarget = codeGenerationTarget,
EnableMarshalMethods = EnableMarshalMethods,
EnableManagedMarshalMethodsLookup = EnableManagedMarshalMethodsLookup,
IsAndroidAssembly = MonoAndroidHelper.IsAndroidAssembly (source),
IsDebug = Debug,
IsFrameworkAssembly = MonoAndroidHelper.IsFrameworkAssembly (source),
Expand Down
115 changes: 109 additions & 6 deletions src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ void Run (bool useMarshalMethods)

// Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture
var nativeCodeGenStates = new ConcurrentDictionary<AndroidTargetArch, NativeCodeGenState> ();
var nativeCodeGenStateObjects = new ConcurrentDictionary<AndroidTargetArch, NativeCodeGenStateObject> ();
NativeCodeGenState? templateCodeGenState = null;

var firstArch = allAssembliesPerArch.First ().Key;
Expand All @@ -169,7 +170,7 @@ void Run (bool useMarshalMethods)
// Pick the "first" one as the one to generate Java code for
var generateJavaCode = arch == firstArch;

(bool success, NativeCodeGenState? state) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, MaybeGetArchAssemblies (userAssembliesPerArch, arch), useMarshalMethods, generateJavaCode);
(bool success, NativeCodeGenState? state, NativeCodeGenStateObject? stateObject) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, MaybeGetArchAssemblies (userAssembliesPerArch, arch), useMarshalMethods, generateJavaCode);

if (!success) {
generateSucceeded = false;
Expand All @@ -181,6 +182,7 @@ void Run (bool useMarshalMethods)
}

nativeCodeGenStates.TryAdd (arch, state);
nativeCodeGenStateObjects.TryAdd (arch, stateObject);
});

// If we hit an error generating the Java code, we should bail out now
Expand All @@ -198,6 +200,21 @@ void Run (bool useMarshalMethods)
// Save NativeCodeGenState for later tasks
Log.LogDebugMessage ($"Saving {nameof (NativeCodeGenState)} to {nameof (NativeCodeGenStateRegisterTaskKey)}");
BuildEngine4.RegisterTaskObjectAssemblyLocal (MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (NativeCodeGenStateRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), nativeCodeGenStates, RegisteredTaskObjectLifetime.Build);

// If we still need the NativeCodeGenState in the <GenerateNativeMarshalMethodSources> task because we're using marshal methods,
// we're going to transfer it to a new object that doesn't require holding open Cecil AssemblyDefinitions.
if (useMarshalMethods) {
//var nativeCodeGenStateObject = MarshalMethodCecilAdapter.GetNativeCodeGenStateCollection (Log, nativeCodeGenStates);
var nativeCodeGenStateCollection = new NativeCodeGenStateCollection ();

foreach (var kvp in nativeCodeGenStateObjects) {
nativeCodeGenStateCollection.States.Add (kvp.Key, kvp.Value);
Log.LogDebugMessage ($"Added NativeCodeGenStateObject for arch: {kvp.Key}, containing {kvp.Value.MarshalMethods.Count} marshal methods");
}

Log.LogDebugMessage ($"Saving {nameof (NativeCodeGenStateObject)} to {nameof (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey)}");
BuildEngine4.RegisterTaskObjectAssemblyLocal (MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), nativeCodeGenStateCollection, RegisteredTaskObjectLifetime.Build);
}
}

internal static Dictionary<string, ITaskItem> MaybeGetArchAssemblies (Dictionary<AndroidTargetArch, Dictionary<string, ITaskItem>> dict, AndroidTargetArch arch)
Expand All @@ -209,7 +226,7 @@ internal static Dictionary<string, ITaskItem> MaybeGetArchAssemblies (Dictionary
return archDict;
}

(bool success, NativeCodeGenState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary<string, ITaskItem> assemblies, Dictionary<string, ITaskItem> userAssemblies, bool useMarshalMethods, bool generateJavaCode)
(bool success, NativeCodeGenState? stubsState, NativeCodeGenStateObject? stateObject) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary<string, ITaskItem> assemblies, Dictionary<string, ITaskItem> userAssemblies, bool useMarshalMethods, bool generateJavaCode)
{
XAAssemblyResolver resolver = MakeResolver (useMarshalMethods, arch, assemblies);
var tdCache = new TypeDefinitionCache ();
Expand All @@ -227,15 +244,62 @@ internal static Dictionary<string, ITaskItem> MaybeGetArchAssemblies (Dictionary
}

if (!success) {
return (false, null);
return (false, null, null);
}

MarshalMethodsCollection? marshalMethodsCollection = null;
NativeCodeGenStateObject? stateObject = null;

if (useMarshalMethods) {
stateObject = new NativeCodeGenStateObject ();

foreach (var assembly in assemblies.Values) {
var marshalMethodXmlFile = MarshalMethodsXmlFile.GetMarshalMethodsXmlFilePath (assembly.ItemSpec);

if (!File.Exists (marshalMethodXmlFile))
continue;

var xml = MarshalMethodsXmlFile.Import (marshalMethodXmlFile);

if (xml is null || xml?.Value.MarshalMethods.Count == 0) {
Log.LogDebugMessage ($"'{marshalMethodXmlFile}' is empty, skipping.");
continue;
}

foreach (var kvp in xml.Value.Value.MarshalMethods) {
if (!stateObject.MarshalMethods.TryGetValue (kvp.Key, out var methods)) {
methods = new List<MarshalMethodEntryObject> ();
stateObject.MarshalMethods.Add (kvp.Key, methods);
}

foreach (var method in kvp.Value) {
// We don't need to add the special case method multiple times
if (methods.Count > 0 && method.IsSpecial)
continue;
methods.Add (method);
}
}
}

//marshalMethodsCollection = MarshalMethodsCollection.FromAssemblies (arch, assemblies.Values.ToList (), resolver, Log);
//marshalMethodsCollection.AddSpecialCaseMethods ();

marshalMethodsCollection = new EmptyMarshalMethodsCollection ();
}

var state = new NativeCodeGenState (arch, tdCache, resolver, allJavaTypes, javaTypesForJCW, marshalMethodsCollection);

if (useMarshalMethods)
marshalMethodsCollection = MarshalMethodsCollection.FromAssemblies (arch, assemblies.Values.ToList (), resolver, Log);
//if (useMarshalMethods) {
// var info = new ManagedMarshalMethodsLookupInfo (Log);

return (true, new NativeCodeGenState (arch, tdCache, resolver, allJavaTypes, javaTypesForJCW, marshalMethodsCollection));
// foreach (var kvp in marshalMethodsCollection.MarshalMethods)
// foreach (var method in kvp.Value)
// info.AddNativeCallbackWrapper (method.NativeCallback);

// state.ManagedMarshalMethodsLookupInfo = info;
//}

return (true, state, stateObject);
}

(List<TypeDefinition> allJavaTypes, List<TypeDefinition> javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolver res, TypeDefinitionCache cache, Dictionary<string, ITaskItem> assemblies, Dictionary<string, ITaskItem> userAssemblies, bool useMarshalMethods)
Expand Down Expand Up @@ -343,4 +407,43 @@ void CompareLists (string name, List<string> list1, List<string> list2)
Log.LogDebugMessage ($"No differences");
}
}

class EmptyMarshalMethodsCollection : MarshalMethodsCollection
{
public override HashSet<AssemblyDefinition> AssembliesWithMarshalMethods => throw new NotSupportedException ();

/// <summary>
/// Marshal methods that have already been rewritten as LLVM marshal methods.
/// </summary>
public override IDictionary<string, IList<ConvertedMarshalMethodEntry>> ConvertedMarshalMethods => throw new NotSupportedException ();

/// <summary>
/// Marshal methods that cannot be rewritten and must be registered dynamically.
/// </summary>
public override List<DynamicallyRegisteredMarshalMethodEntry> DynamicallyRegisteredMarshalMethods => throw new NotSupportedException ();

/// <summary>
/// Marshal methods that can be rewritten as LLVM marshal methods.
/// </summary>
public override IDictionary<string, IList<MarshalMethodEntry>> MarshalMethods => throw new NotSupportedException ();

public EmptyMarshalMethodsCollection ()
{
}

public override void AddSpecialCaseMethods ()
{
throw new NotImplementedException ();
}

public override bool ShouldBeDynamicallyRegistered (TypeDefinition topType, MethodDefinition registeredMethod, MethodDefinition implementedMethod, CustomAttribute registerAttribute)
{
throw new NotImplementedException ();
}

public override bool TypeHasDynamicallyRegisteredMethods (TypeDefinition type)
{
throw new NotImplementedException ();
}
}
}
13 changes: 0 additions & 13 deletions src/Xamarin.Android.Build.Tasks/Tasks/GenerateMainAndroidManifest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ public class GenerateMainAndroidManifest : AndroidTask
public string CodeGenerationTarget { get; set; } = "";
public bool Debug { get; set; }
public bool EmbedAssemblies { get; set; }
public bool EnableMarshalMethods { get; set; }
[Required]
public string IntermediateOutputDirectory { get; set; } = "";
public string []? ManifestPlaceholders { get; set; }
Expand All @@ -52,8 +51,6 @@ public class GenerateMainAndroidManifest : AndroidTask
AndroidRuntime androidRuntime;
JavaPeerStyle codeGenerationTarget;

bool UseMarshalMethods => !Debug && EnableMarshalMethods;

public override bool RunTask ()
{
// Retrieve the stored NativeCodeGenState (and remove it from the cache)
Expand All @@ -74,16 +71,6 @@ public override bool RunTask ()
var additionalProviders = MergeManifest (templateCodeGenState, GenerateJavaStubs.MaybeGetArchAssemblies (userAssembliesPerArch, templateCodeGenState.TargetArch));
GenerateAdditionalProviderSources (templateCodeGenState, additionalProviders);


// If we still need the NativeCodeGenState in the <GenerateNativeMarshalMethodSources> task because we're using marshal methods,
// we're going to transfer it to a new object that doesn't require holding open Cecil AssemblyDefinitions.
if (UseMarshalMethods) {
var nativeCodeGenStateObject = MarshalMethodCecilAdapter.GetNativeCodeGenStateCollection (Log, nativeCodeGenStates);

Log.LogDebugMessage ($"Saving {nameof (NativeCodeGenStateObject)} to {nameof (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey)}");
BuildEngine4.RegisterTaskObjectAssemblyLocal (MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), nativeCodeGenStateObject, RegisteredTaskObjectLifetime.Build);
}

// Dispose the Cecil resolvers so the assemblies are closed.
Log.LogDebugMessage ($"Disposing all {nameof (NativeCodeGenState)}.{nameof (NativeCodeGenState.Resolver)}");

Expand Down
Loading
Loading