Skip to content

Commit d226778

Browse files
SingleAccretionSingleAccretion
authored
[NativeAOT-LLVM] Add Mstat support (#3171)
* Add Mstat support Since we only know the size of the methods once we compile them, introduce a "relocatable object node" concept: 1) We record LLVM methods at object emission time with 0 sizes, recording the location in the IL stream where the size is set. 2) We request the linker to produce a link map. 3) We "relocate" the mstat in an MSBuild task by parsing the aforementioned section together with the link map. * Add a wasmjit-diff change for good measure This argument has proven more harmful than useful - you can just use /p:DebugType=none to test DI-less scenarios.
1 parent 2ef8dae commit d226778

File tree

9 files changed

+218
-13
lines changed

9 files changed

+218
-13
lines changed

‎src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,8 @@ The .NET Foundation licenses this file to you under the MIT license.
524524
strip -no_code_signature_warning -x "$(NativeBinary)"" />
525525
</Target>
526526

527-
<!-- NativeAOT-LLVM: separate target to reduce conflicts -->
527+
<!-- NativeAOT-LLVM: separate target to avoid conflicts -->
528+
<UsingTask TaskName="RelocateMstat" AssemblyFile="$(IlcBuildTasksPath)" />
528529
<Target Name="LinkNativeLlvm"
529530
Inputs="@(NativeObjects);@(NativeLibrary)"
530531
Outputs="$(NativeBinary)"
@@ -611,12 +612,21 @@ The .NET Foundation licenses this file to you under the MIT license.
611612
<CustomLinkerArg Include="-mexec-model=reactor" Condition="'$(NativeLib)' == 'Shared'" />
612613
</ItemGroup>
613614

615+
<ItemGroup Condition="'$(IlcGenerateMstatFile)' == 'true'">
616+
<_LinkMapFilePath Include="$(NativeIntermediateOutputPath)$(TargetName).linkmap" />
617+
<CustomLinkerArg Include="@(_LinkMapFilePath->Replace('\', '/')->'-Wl,--Map=&quot;%(Identity)&quot;')" />
618+
</ItemGroup>
619+
614620
<ItemGroup>
615621
<CustomLinkerArg Include="@(LinkerArg)" />
616622
</ItemGroup>
617623

618624
<WriteLinesToFile File="$(NativeIntermediateOutputPath)link.rsp" Lines="@(CustomLinkerArg)" Overwrite="true" Encoding="utf-8" />
619625
<Exec Command="$(WasmLinkerPath) @$(NativeIntermediateOutputPath)link.rsp $(EmccExtraArgs)" EnvironmentVariables="@(EmscriptenEnvVars)" />
626+
627+
<RelocateMstat Condition="'$(IlcGenerateMstatFile)' == 'true'"
628+
MstatFilePath="$(NativeIntermediateOutputPath)%(ManagedBinary.Filename).mstat"
629+
LinkMapPath="@(_LinkMapFilePath)" />
620630
</Target>
621631

622632
<Target Name="LinkNative"

‎src/coreclr/tools/aot/ILCompiler.Build.Tasks/RelocateMstat.cs‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using Microsoft.Build.Framework;
5+
using Microsoft.Build.Utilities;
6+
using System;
7+
using System.Collections.Generic;
8+
using System.Globalization;
9+
using System.IO;
10+
using System.Reflection.Metadata;
11+
using System.Reflection.PortableExecutable;
12+
using System.Text;
13+
14+
namespace Build.Tasks
15+
{
16+
public class RelocateMstat : Task
17+
{
18+
[Required]
19+
public string MstatFilePath { get; set; }
20+
21+
[Required]
22+
public string LinkMapPath { get; set; }
23+
24+
public override bool Execute()
25+
{
26+
using FileStream stream = File.Open(MstatFilePath, FileMode.Open);
27+
using PEReader reader = new(stream);
28+
29+
int methodsMethodRva = 0;
30+
MetadataReader mdReader = reader.GetMetadataReader();
31+
foreach (MethodDefinitionHandle methodDefHandle in mdReader.MethodDefinitions)
32+
{
33+
MethodDefinition methodDef = mdReader.GetMethodDefinition(methodDefHandle);
34+
if (mdReader.GetString(methodDef.Name) == "Methods")
35+
{
36+
methodsMethodRva = methodDef.RelativeVirtualAddress;
37+
break;
38+
}
39+
}
40+
if (methodsMethodRva == 0)
41+
{
42+
return true;
43+
}
44+
45+
Dictionary<string, int> linkMap = ParseLinkMap();
46+
BlobReader methodsBody = reader.GetSectionData(methodsMethodRva).GetReader();
47+
int ilHeaderSize = ((methodsBody.ReadByte() & 0x3) != 0) ? 12 : 1;
48+
49+
BlobReader relocs = reader.GetSectionData(".szreloc").GetReader();
50+
BlobReader names = reader.GetSectionData(".names").GetReader();
51+
List<(int, string)> relocList = new();
52+
while (relocs.RemainingBytes != 0)
53+
{
54+
int ilOffset = relocs.ReadInt32();
55+
names.Offset = relocs.ReadInt32();
56+
string target = names.ReadSerializedString();
57+
relocList.Add((ilOffset, target));
58+
}
59+
60+
int sectionIndex = reader.PEHeaders.GetContainingSectionIndex(methodsMethodRva);
61+
int sectionVA = reader.PEHeaders.SectionHeaders[sectionIndex].VirtualAddress;
62+
int sectionFileOffset = reader.PEHeaders.SectionHeaders[sectionIndex].PointerToRawData;
63+
int methodsMethodFileOffset = methodsMethodRva - sectionVA + sectionFileOffset;
64+
65+
BinaryWriter writer = new(stream);
66+
foreach (var (ilOffset, target) in relocList)
67+
{
68+
if (linkMap.TryGetValue(target, out int size))
69+
{
70+
writer.Seek(methodsMethodFileOffset + ilHeaderSize + ilOffset, SeekOrigin.Begin);
71+
writer.Write(size);
72+
}
73+
}
74+
return true;
75+
}
76+
77+
private Dictionary<string, int> ParseLinkMap()
78+
{
79+
using StreamReader reader = new(LinkMapPath, Encoding.UTF8);
80+
81+
char[] whitespace = [' '];
82+
bool isCode = false;
83+
Dictionary<string, int> result = new();
84+
while (reader.ReadLine() is string line)
85+
{
86+
string[] parts = line.Split(whitespace, 4, StringSplitOptions.RemoveEmptyEntries);
87+
if (parts.Length >= 4 && parts[3] == "CODE")
88+
{
89+
isCode = true;
90+
continue;
91+
}
92+
if (!isCode)
93+
{
94+
continue;
95+
}
96+
if (parts[3] == "DATA")
97+
{
98+
break;
99+
}
100+
101+
string section = parts[3];
102+
int symbolStart = section.LastIndexOf(":(", StringComparison.Ordinal);
103+
if (symbolStart > 0 && section[section.Length - 1] == ')')
104+
{
105+
symbolStart += 2;
106+
107+
int size = int.Parse(parts[2], NumberStyles.HexNumber, CultureInfo.InvariantCulture);
108+
string symbol = section.Substring(symbolStart, section.Length - symbolStart - 1);
109+
if (result.ContainsKey(symbol))
110+
{
111+
result[symbol] += size;
112+
}
113+
else
114+
{
115+
result.Add(symbol, size);
116+
}
117+
}
118+
}
119+
return result;
120+
}
121+
}
122+
}

‎src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/IObjectDumper.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
namespace ILCompiler.DependencyAnalysis
77
{
8-
public interface IObjectDumper
8+
public partial interface IObjectDumper
99
{
1010
void DumpObjectNode(NodeFactory factory, ObjectNode node, ObjectData objectData);
1111
}

‎src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/MstatObjectDumper.Llvm.cs‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Collections.Generic;
5+
using System.Reflection.Metadata;
6+
using System.Reflection.Metadata.Ecma335;
7+
8+
using ILCompiler.DependencyAnalysis;
9+
using ILCompiler.DependencyAnalysisFramework;
10+
using Internal.IL.Stubs;
11+
using Internal.TypeSystem;
12+
13+
namespace ILCompiler.DependencyAnalysis
14+
{
15+
public partial interface IObjectDumper
16+
{
17+
// In the LLVM backend, compiled method bodies are represented by symbols of which by emission
18+
// time we do not yet know the size. We support dumping them by creating "relocatable" dumps.
19+
void DumpExternalObjectNode(NodeFactory factory, ISymbolNode node);
20+
}
21+
}
22+
23+
namespace ILCompiler
24+
{
25+
public abstract partial class ObjectDumper
26+
{
27+
void IObjectDumper.DumpExternalObjectNode(NodeFactory factory, ISymbolNode node) => DumpExternalObjectNode(factory, node);
28+
29+
protected virtual void DumpExternalObjectNode(NodeFactory factory, ISymbolNode node) { }
30+
31+
private sealed partial class ComposedObjectDumper : ObjectDumper
32+
{
33+
protected override void DumpExternalObjectNode(NodeFactory factory, ISymbolNode node)
34+
{
35+
foreach (var d in _dumpers)
36+
d.DumpExternalObjectNode(factory, node);
37+
}
38+
}
39+
}
40+
41+
public partial class MstatObjectDumper
42+
{
43+
private List<(MethodDesc Method, string MangledName)> _externalMethods = new();
44+
45+
protected override void DumpExternalObjectNode(NodeFactory factory, ISymbolNode node)
46+
{
47+
IMethodBodyNode methodNode = (IMethodBodyNode)node; // We currently only use this for methods.
48+
_externalMethods.Add((methodNode.Method, methodNode.GetMangledName(factory.NameMangler)));
49+
}
50+
51+
private void EmitRelocatableNodes(InstructionEncoder methods)
52+
{
53+
BlobBuilder relocs = new();
54+
foreach (var m in _externalMethods)
55+
{
56+
methods.OpCode(ILOpCode.Ldtoken);
57+
methods.Token(_emitter.EmitMetadataHandleForTypeSystemEntity(m.Method));
58+
methods.OpCode(ILOpCode.Ldc_i4);
59+
int sizeOffset = methods.Offset;
60+
methods.Token(0);
61+
methods.LoadConstantI4(0);
62+
methods.LoadConstantI4(_methodEhInfo.GetValueOrDefault(m.Method));
63+
int mangledNameOffset = AppendMangledName(m.MangledName);
64+
methods.LoadConstantI4(mangledNameOffset);
65+
66+
relocs.WriteInt32(sizeOffset);
67+
relocs.WriteInt32(mangledNameOffset);
68+
}
69+
70+
_emitter.AddPESection(".szreloc", relocs);
71+
}
72+
}
73+
}

‎src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/MstatObjectDumper.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
namespace ILCompiler
2424
{
25-
public class MstatObjectDumper : ObjectDumper
25+
public partial class MstatObjectDumper : ObjectDumper
2626
{
2727
private const int VersionMajor = 2;
2828
private const int VersionMinor = 1;
@@ -161,6 +161,8 @@ internal override void End()
161161
methods.LoadConstantI4(AppendMangledName(m.MangledName));
162162
}
163163

164+
EmitRelocatableNodes(methods);
165+
164166
var blobs = new InstructionEncoder(new BlobBuilder());
165167
foreach (var b in _blobs)
166168
{

‎src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ObjectDumper.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
namespace ILCompiler
1111
{
12-
public abstract class ObjectDumper : IObjectDumper
12+
public abstract partial class ObjectDumper : IObjectDumper
1313
{
1414
internal abstract void Begin();
1515
internal abstract void End();
@@ -47,7 +47,7 @@ public static ObjectDumper Compose(IEnumerable<ObjectDumper> dumpers)
4747
};
4848
}
4949

50-
private sealed class ComposedObjectDumper : ObjectDumper
50+
private sealed partial class ComposedObjectDumper : ObjectDumper
5151
{
5252
private readonly ObjectDumper[] _dumpers;
5353

‎src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,7 @@
487487
<Compile Include="Compiler\MethodImportationErrorProvider.cs" />
488488
<Compile Include="Compiler\InlinedThreadStatics.cs" />
489489
<Compile Include="Compiler\MstatObjectDumper.cs" />
490+
<Compile Include="Compiler\MstatObjectDumper.Llvm.cs" />
490491
<Compile Include="Compiler\NoMetadataBlockingPolicy.cs" />
491492
<Compile Include="Compiler\DependencyAnalysis\SerializedFrozenObjectNode.cs" />
492493
<Compile Include="Compiler\DependencyAnalysis\GCStaticsPreInitDataNode.cs" />

‎src/coreclr/tools/aot/ILCompiler.LLVM/CodeGen/WasmObjectWriter.EmitObject.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,11 @@ public static void EmitObject(string objectFilePath, IEnumerable<DependencyNode>
3535
{
3636
ObjectNode node = depNode as ObjectNode;
3737
if (node == null)
38+
{
39+
if (dumper != null && depNode is LLVMMethodCodeNode methodCodeNode)
40+
dumper.DumpExternalObjectNode(factory, methodCodeNode);
3841
continue;
42+
}
3943

4044
if (node.ShouldSkipEmittingObjectNode(factory))
4145
continue;

‎src/tests/nativeaot/SmokeTests/HelloWasm/wasmjit-diff.ps1‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ Param(
1515
[ValidateSet("Debug","Checked","Release")][string]$Config = "Release",
1616
[ValidateSet("Debug","Checked","Release")][string]$IlcConfig = "Release",
1717
[string][Alias("bt")]$BaseTag = "",
18-
[Nullable[bool]]$DebugSymbols = $null,
1918
[uint]$NumberOfDiffsToShow = 20,
2019
[string]$PassThrough = ""
2120
)
@@ -42,7 +41,6 @@ if ($ShowHelp)
4241
Write-Host " -Config : Test configuration (Debug/Release). Default is Release"
4342
Write-Host " -IlcConfig : ILC configuration (Debug/Checked/Release). Default is Release"
4443
Write-Host " -BaseTag : Suffix to use for the 'base' directory. Default is none"
45-
Write-Host " -DebugSymbols : Whether to build with debug symbols. Default is yes"
4644
Write-Host " -NumberOfDiffsToShow : Number of diffs to show. Default is 20"
4745
Write-Host " -PassThrough : Additional command line to pass directly to 'dotnet'"
4846
Write-Host ""
@@ -58,11 +56,6 @@ if ($ShowHelp)
5856
return
5957
}
6058

61-
if ($DebugSymbols -eq $null)
62-
{
63-
$DebugSymbols = $true
64-
}
65-
6659
$RuntimelabDirectory = [System.IO.Path]::GetFullPath("./../../../../../", $PSScriptRoot)
6760
if (!$RuntimelabDirectory.Replace("\", "/").EndsWith("runtimelab/"))
6861
{
@@ -121,7 +114,7 @@ if ($Build -or $Rebuild)
121114
Write-Host ""
122115
}
123116

124-
$UserBuildArgs = "/p:TargetOS=$OS /p:TargetArchitecture=$Arch /p:IlcConfig=$IlcConfig /p:NativeDebugSymbols=$DebugSymbols -c $Config $PassThrough"
117+
$UserBuildArgs = "/p:TargetOS=$OS /p:TargetArchitecture=$Arch /p:IlcConfig=$IlcConfig -c $Config $PassThrough"
125118
$BuildExpression = "dotnet build $TestProjectPath /t:BuildNativeAot /p:TestBuildMode=nativeaot $UserBuildArgs"
126119
Write-Verbose "Invoking: '$BuildExpression'"
127120
Invoke-Expression $BuildExpression

0 commit comments

Comments
 (0)