Merge pull request #36 from parca-dev/unify_layouts

Unify libc layouts

.github/workflows/build.yml

@@ -41,8 +41,8 @@ jobs:
       - name: Check
         run: make check
 
-      - name: Format
-        run: make format
+      - name: Lint
+        run: make lint
 
       - name: Build
         run: make build

cmd/apkdownload/apkdownload.go

@@ -94,7 +94,7 @@ func main() {
 	}
 
 	interimDir := filepath.Join(*tempDir, *pkgName)
-	if err := os.MkdirAll(interimDir, 0755); err != nil {
+	if err := os.MkdirAll(interimDir, 0o755); err != nil {
 		logger.Error("failed to create temp directory", "err", err)
 		os.Exit(1)
 	}
@@ -109,7 +109,7 @@ func main() {
 	}
 
 	targetDir := filepath.Join(*outputDir, *pkgName)
-	if err := os.MkdirAll(targetDir, 0755); err != nil {
+	if err := os.MkdirAll(targetDir, 0o755); err != nil {
 		logger.Error("failed to create output directory", "err", err)
 		os.Exit(1)
 	}
@@ -136,21 +136,18 @@ type cli struct {
 	logger *slog.Logger
 }
 
-var (
-	allowedVariants = map[string]struct{}{
-		"dbg": {},
-		// "dev": {},
-		// "fts":     {},
-		// "legacy":  {},
-		// "libintl": {},
-		// "locales": {},
-		// "obstack": {},
-		// "utils":   {},
-	}
-)
+var allowedVariants = map[string]struct{}{
+	"dbg": {},
+	// "dev": {},
+	// "fts":     {},
+	// "legacy":  {},
+	// "libintl": {},
+	// "locales": {},
+	// "obstack": {},
+	// "utils":   {},
+}
 
 func (c *cli) list(ctx context.Context, pkgUrl, pkgName string, architectures []string, versionConstraint string) ([]*pkg, error) {
-
 	packages := map[string]*pkg{}
 	for _, arch := range architectures {
 		pkgUrl, err := url.JoinPath(pkgUrl, convertArch(arch))
@@ -371,7 +368,7 @@ func (c *cli) extract(ctx context.Context, packages []*pkg, outputDir string) er
 
 			target := filepath.Join(targetDir, hdr.Name)
 			if hdr.FileInfo().IsDir() {
-				if err := os.MkdirAll(target, 0755); err != nil {
+				if err := os.MkdirAll(target, 0o755); err != nil {
 					return fmt.Errorf("failed to create directory: %w", err)
 				}
 				continue

cmd/debdownload/debdownload.go

@@ -43,12 +43,10 @@ const (
 	DownloadSinglePackageTimeout = 90 * time.Second
 )
 
-var (
-	allowedVariants = map[string]struct{}{
-		"dbg": {},
-		// "dev": {},
-	}
-)
+var allowedVariants = map[string]struct{}{
+	"dbg": {},
+	// "dev": {},
+}
 
 type packageSourceVariant int
 
@@ -123,7 +121,7 @@ func main() {
 	}
 
 	interimDir := filepath.Join(*tempDir, *pkgName)
-	if err := os.MkdirAll(interimDir, 0755); err != nil {
+	if err := os.MkdirAll(interimDir, 0o755); err != nil {
 		logger.Error("failed to create temp directory", "err", err)
 		os.Exit(1)
 	}
@@ -138,7 +136,7 @@ func main() {
 	}
 
 	targetDir := filepath.Join(*outputDir, *pkgName)
-	if err := os.MkdirAll(targetDir, 0755); err != nil {
+	if err := os.MkdirAll(targetDir, 0o755); err != nil {
 		logger.Error("failed to create output directory", "err", err)
 		os.Exit(1)
 	}
@@ -415,7 +413,7 @@ func (c *cli) extract(_ context.Context, packages []*pkg, outputDir string) erro
 
 				target := filepath.Join(targetDir, hdr.Name)
 				if hdr.FileInfo().IsDir() {
-					if err := os.MkdirAll(target, 0755); err != nil {
+					if err := os.MkdirAll(target, 0o755); err != nil {
 						return fmt.Errorf("failed to create directory: %w", err)
 					}
 					continue

cmd/debuginfofind/debuginfofind.go

@@ -25,9 +25,7 @@ func main() {
 	}))
 
 	fSet := ff.NewFlagSet("debdownload")
-	var (
-		debuginfoDir = fSet.String('d', "debuginfo-dir", "", "directory to write the downloaded debuginfo files")
-	)
+	debuginfoDir := fSet.String('d', "debuginfo-dir", "", "directory to write the downloaded debuginfo files")
 	if err := ff.Parse(fSet, os.Args[1:]); err != nil {
 		fmt.Printf("%s\n", ffhelp.Flags(fSet))
 		if !errors.Is(err, ff.ErrHelp) {

cmd/structlayout/structlayout.go

@@ -93,9 +93,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	var (
-		input = fSet.Arg(0)
-	)
+	input := fSet.Arg(0)
 	dwarfData, err := dwarfDataFromELF(input)
 	if err != nil {
 		logger.Error("failed to read DWARF data", "err", err)
@@ -137,7 +135,7 @@ func processAndWriteLayout(dwarfData *dwarf.Data, output string, version string,
 		return fmt.Errorf("failed to extract struct layout from DWARF data: %w", err)
 	}
 
-	if err := os.MkdirAll(filepath.Dir(output), 0755); err != nil {
+	if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil {
 		return fmt.Errorf("failed to create output directory: %w", err)
 	}
 
@@ -174,7 +172,7 @@ func processAndWriteInitialState(dwarfData *dwarf.Data, output string, version s
 		return fmt.Errorf("failed to extract struct layout from DWARF data: %w", err)
 	}
 
-	if err := os.MkdirAll(filepath.Dir(output), 0755); err != nil {
+	if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil {
 		return fmt.Errorf("failed to create output directory: %w", err)
 	}
 

pkg/libc/glibc/datamap.go

@@ -1,6 +1,9 @@
 package glibc
 
-import "github.com/parca-dev/runtime-data/pkg/runtimedata"
+import (
+	"github.com/parca-dev/runtime-data/pkg/libc"
+	"github.com/parca-dev/runtime-data/pkg/runtimedata"
+)
 
 type glibc struct {
 	PThreadSpecific1stblock int64 `offsetof:"pthread.specific_1stblock" yaml:"pthread_specific_1stblock"`
@@ -10,7 +13,7 @@ type glibc struct {
 }
 
 func (g *glibc) Layout() runtimedata.RuntimeData {
-	return &Layout{
+	return &libc.Layout{
 		PThreadSpecific1stblock: g.PThreadSpecific1stblock,
 		PThreadSize:             g.PThreadSize,
 		PThreadKeyData:          g.PThreadKeyData,

pkg/libc/glibc/glibc.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 
 	"github.com/Masterminds/semver/v3"
+	"github.com/parca-dev/runtime-data/pkg/libc"
 	"gopkg.in/yaml.v3"
 )
 
@@ -22,7 +23,7 @@ type Key struct {
 var (
 	//go:embed layout/*/*.yaml
 	generatedLayouts embed.FS
-	structLayouts    = map[Key]*Layout{}
+	structLayouts    = map[Key]*libc.Layout{}
 	once             = &sync.Once{}
 )
 
@@ -34,7 +35,7 @@ func init() {
 	}
 }
 
-func loadLayouts() (map[Key]*Layout, error) {
+func loadLayouts() (map[Key]*libc.Layout, error) {
 	var err error
 	once.Do(func() {
 		entries, err := generatedLayouts.ReadDir(filepath.Join(layoutDir, runtime.GOARCH))
@@ -56,7 +57,7 @@ func loadLayouts() (map[Key]*Layout, error) {
 			if ext != ".yaml" && ext != ".yml" {
 				continue
 			}
-			var lyt Layout
+			var lyt libc.Layout
 			if err = yaml.Unmarshal(data, &lyt); err != nil {
 				return
 			}
@@ -73,7 +74,7 @@ func loadLayouts() (map[Key]*Layout, error) {
 	return structLayouts, err
 }
 
-func getLayoutForArch(v *semver.Version, arch string) (Key, *Layout, error) {
+func getLayoutForArch(v *semver.Version, arch string) (Key, *libc.Layout, error) {
 	entries, err := generatedLayouts.ReadDir(filepath.Join(layoutDir, arch))
 	if err != nil {
 		return Key{}, nil, err
@@ -92,7 +93,7 @@ func getLayoutForArch(v *semver.Version, arch string) (Key, *Layout, error) {
 		if ext != ".yaml" && ext != ".yml" {
 			continue
 		}
-		var lyt Layout
+		var lyt libc.Layout
 		if err = yaml.Unmarshal(data, &lyt); err != nil {
 			return Key{}, nil, err
 		}
@@ -110,7 +111,7 @@ func getLayoutForArch(v *semver.Version, arch string) (Key, *Layout, error) {
 }
 
 // GetLayout returns the layout for the given version.
-func GetLayout(v *semver.Version) (Key, *Layout, error) {
+func GetLayout(v *semver.Version) (Key, *libc.Layout, error) {
 	for k, l := range structLayouts {
 		constr, err := semver.NewConstraint(k.Constraint)
 		if err != nil {
@@ -124,6 +125,6 @@ func GetLayout(v *semver.Version) (Key, *Layout, error) {
 }
 
 // GetLayouts returns all the layouts.
-func GetLayouts() (map[Key]*Layout, error) {
+func GetLayouts() (map[Key]*libc.Layout, error) {
 	return structLayouts, nil
 }

pkg/libc/glibc/glibc_test.go

@@ -5,34 +5,45 @@ import (
 
 	"github.com/Masterminds/semver/v3"
 	"github.com/google/go-cmp/cmp"
+	"github.com/parca-dev/runtime-data/pkg/libc"
 )
 
 func Test_getLayoutForArch(t *testing.T) {
-
 	tests := []struct {
 		name    string
 		v       *semver.Version
 		arch    string
-		want    *Layout
+		want    *libc.Layout
 		wantErr bool
 	}{
 		{
 			name: "2.29.0",
 			v:    semver.MustParse("2.29.0"),
 			arch: "amd64",
-			want: &Layout{
-				PThreadSpecific1stblock: 784,
+			want: &libc.Layout{
+				PThreadSpecific1stblock: 0x310,
 				PThreadSize:             2304,
 				PThreadKeyData:          0x08,
 				PThreadKeyDataSize:      0x10,
 			},
 		},
+		{
+			name: "2.39.0",
+			v:    semver.MustParse("2.39.0"),
+			arch: "amd64",
+			want: &libc.Layout{
+				PThreadSpecific1stblock: 0x310,
+				PThreadSize:             2368,
+				PThreadKeyData:          0x08,
+				PThreadKeyDataSize:      0x10,
+			},
+		},
 		{
 			name: "2.29.0",
 			v:    semver.MustParse("2.29.0"),
 			arch: "arm64",
-			want: &Layout{
-				PThreadSpecific1stblock: 272,
+			want: &libc.Layout{
+				PThreadSpecific1stblock: 0x110,
 				PThreadSize:             1792,
 				PThreadKeyData:          0x08,
 				PThreadKeyDataSize:      0x10,
@@ -42,8 +53,8 @@ func Test_getLayoutForArch(t *testing.T) {
 			name: "2.37.0",
 			v:    semver.MustParse("2.37.0"),
 			arch: "amd64",
-			want: &Layout{
-				PThreadSpecific1stblock: 784,
+			want: &libc.Layout{
+				PThreadSpecific1stblock: 0x310,
 				PThreadSize:             2368,
 				PThreadKeyData:          0x08,
 				PThreadKeyDataSize:      0x10,
@@ -53,8 +64,8 @@ func Test_getLayoutForArch(t *testing.T) {
 			name: "2.37.0",
 			v:    semver.MustParse("2.37.0"),
 			arch: "arm64",
-			want: &Layout{
-				PThreadSpecific1stblock: 272,
+			want: &libc.Layout{
+				PThreadSpecific1stblock: 0x110,
 				PThreadSize:             1856,
 				PThreadKeyData:          0x08,
 				PThreadKeyDataSize:      0x10,
@@ -68,7 +79,7 @@ func Test_getLayoutForArch(t *testing.T) {
 				t.Errorf("getLayoutForArch(%s) on %s error = %v, wantErr %v", tt.name, tt.arch, err, tt.wantErr)
 				return
 			}
-			if diff := cmp.Diff(tt.want, got, cmp.AllowUnexported(Layout{})); diff != "" {
+			if diff := cmp.Diff(tt.want, got, cmp.AllowUnexported(libc.Layout{})); diff != "" {
 				t.Errorf("getLayoutForArch(%s) on %s mismatch (-want +got):\n%s", tt.name, tt.arch, diff)
 			}
 		})

pkg/libc/glibc/layout.go → pkg/libc/layout.go

@@ -1,4 +1,4 @@
-package glibc
+package libc
 
 import (
 	"bytes"
@@ -9,8 +9,8 @@ import (
 )
 
 type Layout struct {
-	PThreadSpecific1stblock int64 `yaml:"pthread_specific_1stblock"`
 	PThreadSize             int64 `yaml:"pthread_size"`
+	PThreadSpecific1stblock int64 `yaml:"pthread_specific_1stblock"`
 	PThreadKeyData          int64 `yaml:"pthread_key_data"`
 	PThreadKeyDataSize      int64 `yaml:"pthread_key_data_size"`
 }

pkg/libc/musl/datamap.go

@@ -1,16 +1,22 @@
 package musl
 
-import "github.com/parca-dev/runtime-data/pkg/runtimedata"
+import (
+	"github.com/parca-dev/runtime-data/pkg/libc"
+	"github.com/parca-dev/runtime-data/pkg/runtimedata"
+)
 
 type musl struct {
 	PThreadSize int64 `sizeof:"__pthread" yaml:"pthread_size"`
 	PThreadTSD  int64 `offsetof:"__pthread.tsd" yaml:"pthread_tsd"`
 }
 
 func (m *musl) Layout() runtimedata.RuntimeData {
-	return &Layout{
-		PthreadSize: m.PThreadSize,
-		PthreadTSD:  m.PThreadTSD,
+	return &libc.Layout{
+		PThreadSize:             m.PThreadSize,
+		PThreadSpecific1stblock: m.PThreadTSD,
+		// TODO(kakkoyun): Extract.
+		PThreadKeyData:     8,
+		PThreadKeyDataSize: 16,
 	}
 }