From bf635eaace603955340b6e5e3b427ffe94af76c1 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Thu, 22 Feb 2024 23:09:44 +0100 Subject: [PATCH] Add musl offsets Signed-off-by: Kemal Akkoyun --- .github/workflows/generate-musl.yml | 74 +++ .gitignore | 1 + Makefile | 18 +- README.md | 21 +- cmd/apkdownload/apkdownload.go | 431 ++++++++++++++++++ cmd/structlayout/structlayout.go | 11 +- pkg/libc/musl.go | 3 - pkg/libc/musl/datamap.go | 19 + pkg/libc/musl/layout.go | 25 + .../musl/layout/amd64/1.1.11 - 1.1.15.yaml | 2 + .../musl/layout/amd64/1.1.16 - 1.1.19.yaml | 2 + .../musl/layout/amd64/1.1.22 - 1.1.24.yaml | 2 + pkg/libc/musl/layout/amd64/1.2.2 - 1.2.4.yaml | 2 + pkg/libc/musl/layout/amd64/= 1.1.20.yaml | 2 + pkg/libc/musl/layout/amd64/= 1.1.4.yaml | 2 + pkg/libc/musl/layout/amd64/= 1.1.5.yaml | 2 + .../musl/layout/arm64/1.1.16 - 1.1.19.yaml | 2 + .../musl/layout/arm64/1.1.22 - 1.1.24.yaml | 2 + pkg/libc/musl/layout/arm64/1.2.2 - 1.2.4.yaml | 2 + pkg/libc/musl/layout/arm64/= 1.1.15.yaml | 2 + pkg/libc/musl/layout/arm64/= 1.1.20.yaml | 2 + pkg/libc/musl/musl.go | 129 ++++++ pkg/libc/musl/musl_test.go | 68 +++ scripts/download/musl.sh | 86 ++++ scripts/mergelayout/musl.sh | 36 ++ scripts/structlayout/musl.sh | 33 ++ 26 files changed, 968 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/generate-musl.yml create mode 100644 cmd/apkdownload/apkdownload.go delete mode 100644 pkg/libc/musl.go create mode 100644 pkg/libc/musl/datamap.go create mode 100644 pkg/libc/musl/layout.go create mode 100644 pkg/libc/musl/layout/amd64/1.1.11 - 1.1.15.yaml create mode 100644 pkg/libc/musl/layout/amd64/1.1.16 - 1.1.19.yaml create mode 100644 pkg/libc/musl/layout/amd64/1.1.22 - 1.1.24.yaml create mode 100644 pkg/libc/musl/layout/amd64/1.2.2 - 1.2.4.yaml create mode 100644 pkg/libc/musl/layout/amd64/= 1.1.20.yaml create mode 100644 pkg/libc/musl/layout/amd64/= 1.1.4.yaml create mode 100644 pkg/libc/musl/layout/amd64/= 1.1.5.yaml create mode 100644 pkg/libc/musl/layout/arm64/1.1.16 - 1.1.19.yaml create mode 100644 pkg/libc/musl/layout/arm64/1.1.22 - 1.1.24.yaml create mode 100644 pkg/libc/musl/layout/arm64/1.2.2 - 1.2.4.yaml create mode 100644 pkg/libc/musl/layout/arm64/= 1.1.15.yaml create mode 100644 pkg/libc/musl/layout/arm64/= 1.1.20.yaml create mode 100644 pkg/libc/musl/musl.go create mode 100644 pkg/libc/musl/musl_test.go create mode 100755 scripts/download/musl.sh create mode 100755 scripts/mergelayout/musl.sh create mode 100755 scripts/structlayout/musl.sh diff --git a/.github/workflows/generate-musl.yml b/.github/workflows/generate-musl.yml new file mode 100644 index 0000000..e685c23 --- /dev/null +++ b/.github/workflows/generate-musl.yml @@ -0,0 +1,74 @@ +name: Generate musl +on: + workflow_call: + push: + branches: + - main + - release + paths: + - ".github/workflows/generate-musl.yml" + - "pkg/libc/musl" + - "scripts" + - "Makefile" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + generate-and-create-branch: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out the code + uses: actions/checkout@v4 + + - name: Install devbox + uses: jetpack-io/devbox-install-action@v0.8.0 + with: + enable-cache: true + + - name: Setup devbox + run: devbox run -- echo "done!" + + - name: Load devbox shellenv + uses: HatsuneMiku3939/direnv-action@v1 + with: + direnvVersion: 2.32.3 + + - name: Set up Go tool cache + uses: actions/cache@v4 + with: + path: ~/.devbox/go + key: devbox-go-tools.cache-${{ runner.os }}-${{ runner.arch }} + + - name: Build + run: make build + + - name: Set up cache for downloaded files + uses: actions/cache@v4 + with: + path: workspace-musl + key: musl-downloaded-${{ runner.os }}-${{ matrix.arch }} + restore-keys: | + musl-downloaded-${{ runner.os }}-${{ matrix.arch }} + + - name: Generate musl Offsets + run: | + TEMP_DIR=workspace-musl make generate/musl + + # If there are no changes (i.e. no diff exists with the checked-out base branch), + # no pull request will be created and the action exits silently. + - name: Create a pull-request + if: github.event_name != 'pull_request' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "chore: update musl layouts" + title: "chore: Update musl layouts" + branch: update-offsets-musl-${{ github.run_number }} + add-paths: pkg/libc/musl/layout + base: main + labels: chore + draft: false + delete-branch: true diff --git a/.gitignore b/.gitignore index cdcfeb0..e803938 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ tmp /mergelayout /debdownload /debuginfofind +/apkdownload /main diff --git a/Makefile b/Makefile index c104fef..ced290f 100644 --- a/Makefile +++ b/Makefile @@ -14,15 +14,18 @@ mergelayout: cmd/mergelayout/mergelayout.go $(filter-out *_test.go,$(GO_SRC)) debdownload: cmd/debdownload/debdownload.go $(filter-out *_test.go,$(GO_SRC)) go build -o $@ $< +apkdownload: cmd/apkdownload/apkdownload.go $(filter-out *_test.go,$(GO_SRC)) + go build -o $@ $< + debuginfofind: cmd/debuginfofind/debuginfofind.go $(filter-out *_test.go,$(GO_SRC)) go build -o $@ $< .PHONY: build -build: structlayout mergelayout debdownload debuginfofind +build: structlayout mergelayout debdownload apkdownload debuginfofind go build ./... .PHONY: generate -generate: build generate/python generate/ruby generate/glibc +generate: build generate/python generate/ruby generate/glibc generate/musl .PHONY: generate/python generate/python: @@ -42,6 +45,12 @@ generate/glibc: ./scripts/structlayout/glibc.sh ./scripts/mergelayout/glibc.sh +.PHONY: generate/musl +generate/musl: + ./scripts/download/musl.sh + ./scripts/structlayout/musl.sh + ./scripts/mergelayout/musl.sh + .PHONY: clean clean: rm -rf target @@ -97,10 +106,13 @@ $(TMPDIR)/mergelayout-help.txt: $(TMPDIR) ./cmd/mergelayout/mergelayout.go $(TMPDIR)/debdownload-help.txt: $(TMPDIR) ./cmd/debdownload/debdownload.go go run ./cmd/debdownload/debdownload.go -h > $@ 2>&1 +$(TMPDIR)/apkdownload-help.txt: $(TMPDIR) ./cmd/apkdownload/apkdownload.go + go run ./cmd/apkdownload/apkdownload.go -h > $@ 2>&1 + $(TMPDIR)/debuginfofind-help.txt: $(TMPDIR) ./cmd/debuginfofind/debuginfofind.go go run ./cmd/debuginfofind/debuginfofind.go -h > $@ 2>&1 .PHONY: README.md -README.md: $(TMPDIR)/structlayout-help.txt $(TMPDIR)/mergelayout-help.txt $(TMPDIR)/debdownload-help.txt $(TMPDIR)/debuginfofind-help.txt +README.md: $(TMPDIR)/structlayout-help.txt $(TMPDIR)/mergelayout-help.txt $(TMPDIR)/debdownload-help.txt $(TMPDIR)/debuginfofind-help.txt $(TMPDIR)/apkdownload-help.txt go run github.com/campoy/embedmd/v2@latest -w README.md devbox generate readme CONTRIBUTING.md diff --git a/README.md b/README.md index 64cd839..351455f 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,9 @@ flags: -output string output directory to write the layout file -r string - name of the pre-defined runtime, e.g. python, ruby, libc (shorthand) + name of the pre-defined runtime, e.g. python, ruby, libc, musl (shorthand) -runtime string - name of the pre-defined runtime, e.g. python, ruby, libc + name of the pre-defined runtime, e.g. python, ruby, libc, musl -v string version of the runtime that the layout to generate, e.g. 3.9.5 (shorthand) -version string @@ -139,6 +139,23 @@ FLAGS ``` + +### apkdownload +[embedmd]:# (tmp/apkdownload-help.txt) +```txt +NAME + apkdownload + +FLAGS + -o, --output STRING output directory to write the downloaded apk files (default: tmp/bin) + -t, --temp-dir STRING temporary directory to download deb files (default: tmp/apk) + -u, --url STRING URL to download apk files from + -p, --package STRING package name to download + -a, --arch STRING architectures to download + -c, --constraint STRING version constraints to download + +``` + ### debuginfofind [embedmd]:# (tmp/debuginfofind-help.txt) ```txt diff --git a/cmd/apkdownload/apkdownload.go b/cmd/apkdownload/apkdownload.go new file mode 100644 index 0000000..9ab5c1d --- /dev/null +++ b/cmd/apkdownload/apkdownload.go @@ -0,0 +1,431 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "github.com/Masterminds/semver/v3" + "github.com/peterbourgon/ff/v4" + "github.com/peterbourgon/ff/v4/ffhelp" + "golang.org/x/exp/maps" + "golang.org/x/net/html" +) + +const ( + // https://dl-cdn.alpinelinux.org/alpine/MIRRORS.txt + DefaultBaseURL = "https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/" + // FetchListTimeout is the timeout to fetch the list of packages. + FetchListTimeout = 30 * time.Second + + // DownloadSinglePackageTimeout is the timeout to download a single package. + DownloadSinglePackageTimeout = 90 * time.Second +) + +func main() { + logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelDebug, + })) + + fSet := ff.NewFlagSet("apkdownload") + var ( + outputDir = fSet.String('o', "output", "tmp/bin", "output directory to write the downloaded apk files") + tempDir = fSet.String('t', "temp-dir", "tmp/apk", "temporary directory to download deb files") + url = fSet.String('u', "url", "", "URL to download apk files from") + pkgName = fSet.String('p', "package", "", "package name to download") + architectures = fSet.StringList('a', "arch", "architectures to download") + versionConstraint = fSet.String('c', "constraint", "", "version constraints to download") + ) + if err := ff.Parse(fSet, os.Args[1:]); err != nil { + fmt.Printf("%s\n", ffhelp.Flags(fSet)) + if !errors.Is(err, ff.ErrHelp) { + fmt.Printf("err=%v\n", err) + os.Exit(1) + } + os.Exit(0) + } + + if *outputDir == "" { + logger.Error("output directory is required") + os.Exit(1) + } + + if *tempDir == "" { + *tempDir = os.TempDir() + } + + if *url == "" { + *url = DefaultBaseURL + } + + if *pkgName == "" { + logger.Error("package name is required") + os.Exit(1) + } + + if len(*architectures) == 0 { + *architectures = []string{"amd64", "arm64"} + } + + cli := &cli{ + logger: logger, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + packages, err := cli.list(ctx, *url, *pkgName, *architectures, *versionConstraint) + if err != nil { + logger.Error("failed to list packages", "err", err) + os.Exit(1) + } + + interimDir := filepath.Join(*tempDir, *pkgName) + if err := os.MkdirAll(interimDir, 0755); err != nil { + logger.Error("failed to create temp directory", "err", err) + os.Exit(1) + } + + sort.Slice(packages, func(i, j int) bool { + return packages[i].version.GreaterThan(packages[j].version) + }) + + if err := cli.download(ctx, packages, interimDir); err != nil { + logger.Error("failed to download packages", "err", err) + os.Exit(1) + } + + targetDir := filepath.Join(*outputDir, *pkgName) + if err := os.MkdirAll(targetDir, 0755); err != nil { + logger.Error("failed to create output directory", "err", err) + os.Exit(1) + } + + if err := cli.extract(ctx, packages, targetDir); err != nil { + logger.Error("failed to extract packages", "err", err) + os.Exit(1) + } + + logger.Info("downloaded packages", "outputDir", *outputDir) +} + +type pkg struct { + link string + name string + variant string + version *semver.Version + arch string + + downloadedArchive string +} + +type cli struct { + logger *slog.Logger +} + +var ( + allowedVariants = map[string]struct{}{ + "dbg": {}, + // "dev": {}, + // "fts": {}, + // "legacy": {}, + // "libintl": {}, + // "locales": {}, + // "obstack": {}, + // "utils": {}, + } +) + +func (c *cli) list(ctx context.Context, pkgUrl, pkgName string, architectures []string, versionConstraint string) ([]*pkg, error) { + + packages := map[string]*pkg{} + for _, arch := range architectures { + pkgUrl, err := url.JoinPath(pkgUrl, convertArch(arch)) + if err != nil { + return nil, fmt.Errorf("failed to join URL: %w", err) + } + c.logger.Info("listing packages", "url", pkgUrl) + + ctx, cancel := context.WithTimeout(ctx, FetchListTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pkgUrl, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode/100 != 2 { + if resp.StatusCode == http.StatusNotFound { + c.logger.Info("this version probably does not have this architecture supported", "arch", arch, "url", pkgUrl) + continue + } + return nil, fmt.Errorf("unexpected status code (%s): %d", pkgUrl, resp.StatusCode) + } + + doc, err := html.Parse(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to parse HTML: %w", err) + } + + matcher := regexp.MustCompile( + fmt.Sprintf(`%s(-.*)?-([0-9]\.[0-9]\.[0-9][0-9]?).*\.apk`, pkgName), + ) + c.logger.Info("matcher", "pattern", matcher.String()) + + key := func(p *pkg) string { + return strings.Join([]string{p.name, p.variant, shortVersion(p.version), p.arch}, "-") + } + + var process func(*html.Node) + process = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + for _, a := range n.Attr { + if a.Key == "href" { + if matches := matcher.FindStringSubmatch(a.Val); len(matches) > 0 { + version := semver.MustParse(strings.ReplaceAll(matches[2], "~", "+")) + if versionConstraint != "" { + match, reasons := mustConstraint(semver.NewConstraint(versionConstraint)).Validate(version) + if !match { + c.logger.Info("version does not match", "version", version, "reasons", reasons) + c.logger.Info("see: https://github.com/Masterminds/semver?tab=readme-ov-file#checking-version-constraints") + continue + } + } + variant := strings.TrimPrefix(matches[1], "-") + if variant != "" { + if _, ok := allowedVariants[variant]; !ok { + continue + } + } + p := &pkg{ + link: must(url.JoinPath(pkgUrl, a.Val)), + name: pkgName, + variant: variant, + version: version, + arch: arch, + } + if oldPkg, ok := packages[key(p)]; ok { + if p.version.GreaterThan(oldPkg.version) { + c.logger.Info("found newer version", "old", oldPkg.version, "new", p.version) + packages[key(p)] = p + } + continue + } + + packages[key(p)] = p + } + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + process(c) + } + } + process(doc) + } + return maps.Values(packages), nil +} + +func shortVersion(v *semver.Version) string { + return fmt.Sprintf("%d.%d", v.Major(), v.Minor()) +} + +func (c *cli) download(ctx context.Context, packages []*pkg, tempDir string) error { + c.logger.Info("downloading packages", "tempDir", tempDir) + + for _, p := range packages { + var ( + target string + version = p.version.String() + ) + if p.variant != "" { + target = filepath.Join(tempDir, fmt.Sprintf("%s-%s_%s_%s.apk", p.name, p.variant, version, p.arch)) + } else { + target = filepath.Join(tempDir, fmt.Sprintf("%s_%s_%s.apk", p.name, version, p.arch)) + } + if _, err := os.Stat(target); err == nil { + c.logger.Info("file already exists", "file", target) + p.downloadedArchive = target + continue + } + + c.logger.Info("downloading package", "link", p.link, "target", target) + + ctx, cancel := context.WithTimeout(ctx, DownloadSinglePackageTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.link, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode/100 != 2 { + return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + f, err := os.Create(target) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer f.Close() + + if _, err := io.Copy(f, resp.Body); err != nil { + return fmt.Errorf("failed to copy file: %w", err) + } + + p.downloadedArchive = target + } + return nil +} + +func (c *cli) extract(ctx context.Context, packages []*pkg, outputDir string) error { + c.logger.Info("extracting packages", "outputDir", outputDir) + + for _, p := range packages { + if p.downloadedArchive == "" { + continue + } + + if !strings.HasSuffix(p.downloadedArchive, ".apk") { + continue + } + + f, err := os.Open(p.downloadedArchive) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + defer f.Close() + + c.logger.Info("extracting package", "file", f.Name()) + + var variant string + if p.variant != "" { + variant = p.variant + } else { + variant = "main" + } + shortVersion := fmt.Sprintf("%d.%d.%d", p.version.Major(), p.version.Minor(), p.version.Patch()) + targetDir := filepath.Join(outputDir, p.arch, shortVersion, variant) + if _, err := os.Stat(targetDir); err == nil { + c.logger.Info("file already exists", "file", targetDir) + continue + } + + buf := new(bytes.Buffer) + if _, err := buf.ReadFrom(f); err != nil { + return fmt.Errorf("failed to read file: %w", err) + } + + b := buf.Bytes() + var offsets []int + for i := range b { + if readGZIPHeader(b[i:]) { + offsets = append(offsets, i) + } + } + block := b[offsets[2]:] + br := bytes.NewReader(block) + r, err := gzip.NewReader(br) + if err != nil { + return err + } + defer r.Close() + + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("failed to read next entry: %w", err) + } + if hdr == nil { + break + } + + target := filepath.Join(targetDir, hdr.Name) + if hdr.FileInfo().IsDir() { + if err := os.MkdirAll(target, 0755); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + continue + } + + f, err := os.Create(target) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer f.Close() + + if _, err := io.Copy(f, tr); err != nil { + return fmt.Errorf("failed to copy file: %w", err) + } + } + } + return nil +} + +// Signature bytes for finding GZIP header +const ( + GzipID1 = 0x1f + GzipID2 = 0x8b + GzipDeflate = 8 +) + +// readGZIPHeader reads the header of a gzip file if found. +func readGZIPHeader(buf []byte) bool { + if buf[0] != GzipID1 || buf[1] != GzipID2 || buf[2] != GzipDeflate { + return false + } + return true +} + +func convertArch(arch string) string { + switch arch { + case "amd64": + return "x86_64" + case "arm64": + return "aarch64" + } + return arch +} + +func must(u string, err error) string { + if err != nil { + panic(err) + } + return u +} + +func mustConstraint(c *semver.Constraints, err error) *semver.Constraints { + if err != nil { + panic(err) + } + return c +} diff --git a/cmd/structlayout/structlayout.go b/cmd/structlayout/structlayout.go index 469dd0b..bcf4b13 100644 --- a/cmd/structlayout/structlayout.go +++ b/cmd/structlayout/structlayout.go @@ -15,6 +15,7 @@ import ( "github.com/parca-dev/runtime-data/pkg/datamap" "github.com/parca-dev/runtime-data/pkg/libc/glibc" + "github.com/parca-dev/runtime-data/pkg/libc/musl" "github.com/parca-dev/runtime-data/pkg/python" "github.com/parca-dev/runtime-data/pkg/ruby" "github.com/parca-dev/runtime-data/pkg/runtimedata" @@ -33,8 +34,8 @@ func main() { version string givenOutputDir string ) - fSet.StringVar(&runtime, "runtime", "", "name of the pre-defined runtime, e.g. python, ruby, libc") - fSet.StringVar(&runtime, "r", "", "name of the pre-defined runtime, e.g. python, ruby, libc (shorthand)") + fSet.StringVar(&runtime, "runtime", "", "name of the pre-defined runtime, e.g. python, ruby, libc, musl") + fSet.StringVar(&runtime, "r", "", "name of the pre-defined runtime, e.g. python, ruby, libc, musl (shorthand)") fSet.StringVar(&version, "version", "", "version of the runtime that the layout to generate, e.g. 3.9.5") fSet.StringVar(&version, "v", "", "version of the runtime that the layout to generate, e.g. 3.9.5 (shorthand)") fSet.StringVar(&givenOutputDir, "output", "", "output directory to write the layout file") @@ -74,11 +75,15 @@ func main() { outputDir = "pkg/ruby" } case "glibc": - // TODO(kakkoyun): Change depending on the libc implementation. e.g musl, glibc, etc. layoutMap = glibc.DataMapForLayout(version) if outputDir == "" { outputDir = "pkg/libc/glibc/layout" } + case "musl": + layoutMap = musl.DataMapForLayout(version) + if outputDir == "" { + outputDir = "pkg/libc/musl/layout" + } default: logger.Error("invalid offset map module", "mod", runtime) os.Exit(1) diff --git a/pkg/libc/musl.go b/pkg/libc/musl.go deleted file mode 100644 index 6e62cb4..0000000 --- a/pkg/libc/musl.go +++ /dev/null @@ -1,3 +0,0 @@ -package libc - - diff --git a/pkg/libc/musl/datamap.go b/pkg/libc/musl/datamap.go new file mode 100644 index 0000000..d96e77d --- /dev/null +++ b/pkg/libc/musl/datamap.go @@ -0,0 +1,19 @@ +package musl + +import "github.com/parca-dev/runtime-data/pkg/runtimedata" + +type musl struct { + PThreadSize int64 `sizeof:"__pthread" yaml:"pthread_size"` + PThreadTSD int64 `offsetof:"__pthread.tsd" yaml:"pthread_tsd"` +} + +func (m *musl) Layout() runtimedata.RuntimeData { + return &Layout{ + PthreadSize: m.PThreadSize, + PthreadTSD: m.PThreadTSD, + } +} + +func DataMapForLayout(version string) runtimedata.LayoutMap { + return &musl{} +} diff --git a/pkg/libc/musl/layout.go b/pkg/libc/musl/layout.go new file mode 100644 index 0000000..2244130 --- /dev/null +++ b/pkg/libc/musl/layout.go @@ -0,0 +1,25 @@ +package musl + +import ( + "bytes" + "encoding/binary" + "unsafe" + + "github.com/parca-dev/runtime-data/pkg/byteorder" +) + +type Layout struct { + PthreadSize int64 `yaml:"pthread_size"` + PthreadTSD int64 `yaml:"pthread_tsd"` +} + +func (m Layout) Data() ([]byte, error) { + buf := new(bytes.Buffer) + buf.Grow(int(unsafe.Sizeof(&m))) + + if err := binary.Write(buf, byteorder.GetHostByteOrder(), &m); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/pkg/libc/musl/layout/amd64/1.1.11 - 1.1.15.yaml b/pkg/libc/musl/layout/amd64/1.1.11 - 1.1.15.yaml new file mode 100644 index 0000000..6d6d41e --- /dev/null +++ b/pkg/libc/musl/layout/amd64/1.1.11 - 1.1.15.yaml @@ -0,0 +1,2 @@ +pthread_size: 336 +pthread_tsd: 152 diff --git a/pkg/libc/musl/layout/amd64/1.1.16 - 1.1.19.yaml b/pkg/libc/musl/layout/amd64/1.1.16 - 1.1.19.yaml new file mode 100644 index 0000000..d867693 --- /dev/null +++ b/pkg/libc/musl/layout/amd64/1.1.16 - 1.1.19.yaml @@ -0,0 +1,2 @@ +pthread_size: 280 +pthread_tsd: 152 diff --git a/pkg/libc/musl/layout/amd64/1.1.22 - 1.1.24.yaml b/pkg/libc/musl/layout/amd64/1.1.22 - 1.1.24.yaml new file mode 100644 index 0000000..50b1d07 --- /dev/null +++ b/pkg/libc/musl/layout/amd64/1.1.22 - 1.1.24.yaml @@ -0,0 +1,2 @@ +pthread_size: 224 +pthread_tsd: 136 diff --git a/pkg/libc/musl/layout/amd64/1.2.2 - 1.2.4.yaml b/pkg/libc/musl/layout/amd64/1.2.2 - 1.2.4.yaml new file mode 100644 index 0000000..96ed5ea --- /dev/null +++ b/pkg/libc/musl/layout/amd64/1.2.2 - 1.2.4.yaml @@ -0,0 +1,2 @@ +pthread_size: 200 +pthread_tsd: 128 diff --git a/pkg/libc/musl/layout/amd64/= 1.1.20.yaml b/pkg/libc/musl/layout/amd64/= 1.1.20.yaml new file mode 100644 index 0000000..9d9a553 --- /dev/null +++ b/pkg/libc/musl/layout/amd64/= 1.1.20.yaml @@ -0,0 +1,2 @@ +pthread_size: 240 +pthread_tsd: 152 diff --git a/pkg/libc/musl/layout/amd64/= 1.1.4.yaml b/pkg/libc/musl/layout/amd64/= 1.1.4.yaml new file mode 100644 index 0000000..82519d5 --- /dev/null +++ b/pkg/libc/musl/layout/amd64/= 1.1.4.yaml @@ -0,0 +1,2 @@ +pthread_size: 288 +pthread_tsd: 144 diff --git a/pkg/libc/musl/layout/amd64/= 1.1.5.yaml b/pkg/libc/musl/layout/amd64/= 1.1.5.yaml new file mode 100644 index 0000000..b6409f2 --- /dev/null +++ b/pkg/libc/musl/layout/amd64/= 1.1.5.yaml @@ -0,0 +1,2 @@ +pthread_size: 296 +pthread_tsd: 144 diff --git a/pkg/libc/musl/layout/arm64/1.1.16 - 1.1.19.yaml b/pkg/libc/musl/layout/arm64/1.1.16 - 1.1.19.yaml new file mode 100644 index 0000000..d867693 --- /dev/null +++ b/pkg/libc/musl/layout/arm64/1.1.16 - 1.1.19.yaml @@ -0,0 +1,2 @@ +pthread_size: 280 +pthread_tsd: 152 diff --git a/pkg/libc/musl/layout/arm64/1.1.22 - 1.1.24.yaml b/pkg/libc/musl/layout/arm64/1.1.22 - 1.1.24.yaml new file mode 100644 index 0000000..50b1d07 --- /dev/null +++ b/pkg/libc/musl/layout/arm64/1.1.22 - 1.1.24.yaml @@ -0,0 +1,2 @@ +pthread_size: 224 +pthread_tsd: 136 diff --git a/pkg/libc/musl/layout/arm64/1.2.2 - 1.2.4.yaml b/pkg/libc/musl/layout/arm64/1.2.2 - 1.2.4.yaml new file mode 100644 index 0000000..bb23f1e --- /dev/null +++ b/pkg/libc/musl/layout/arm64/1.2.2 - 1.2.4.yaml @@ -0,0 +1,2 @@ +pthread_size: 200 +pthread_tsd: 112 diff --git a/pkg/libc/musl/layout/arm64/= 1.1.15.yaml b/pkg/libc/musl/layout/arm64/= 1.1.15.yaml new file mode 100644 index 0000000..6d6d41e --- /dev/null +++ b/pkg/libc/musl/layout/arm64/= 1.1.15.yaml @@ -0,0 +1,2 @@ +pthread_size: 336 +pthread_tsd: 152 diff --git a/pkg/libc/musl/layout/arm64/= 1.1.20.yaml b/pkg/libc/musl/layout/arm64/= 1.1.20.yaml new file mode 100644 index 0000000..9d9a553 --- /dev/null +++ b/pkg/libc/musl/layout/arm64/= 1.1.20.yaml @@ -0,0 +1,2 @@ +pthread_size: 240 +pthread_tsd: 152 diff --git a/pkg/libc/musl/musl.go b/pkg/libc/musl/musl.go new file mode 100644 index 0000000..c6261cb --- /dev/null +++ b/pkg/libc/musl/musl.go @@ -0,0 +1,129 @@ +package musl + +import ( + "embed" + "errors" + "path/filepath" + "runtime" + "strings" + "sync" + + "github.com/Masterminds/semver/v3" + "gopkg.in/yaml.v2" +) + +const layoutDir = "layout" + +type Key struct { + Index int + Constraint string +} + +var ( + //go:embed layout/*/*.yaml + generatedLayouts embed.FS + structLayouts = map[Key]*Layout{} + once = &sync.Once{} +) + +func init() { + var err error + structLayouts, err = loadLayouts() + if err != nil { + panic(err) + } +} + +func loadLayouts() (map[Key]*Layout, error) { + var err error + once.Do(func() { + entries, err := generatedLayouts.ReadDir(filepath.Join(layoutDir, runtime.GOARCH)) + if err != nil { + return + } + var i int + for _, entry := range entries { + if entry.IsDir() { + continue + } + var data []byte + data, err = generatedLayouts.ReadFile(filepath.Join(layoutDir, runtime.GOARCH, entry.Name())) + if err != nil { + return + } + ext := filepath.Ext(entry.Name()) + // Filter out non-yaml files. + if ext != ".yaml" && ext != ".yml" { + continue + } + var lyt Layout + if err = yaml.Unmarshal(data, &lyt); err != nil { + return + } + rawConstraint := strings.TrimSuffix(entry.Name(), ext) + constr, err := semver.NewConstraint(rawConstraint) + if err != nil { + return + } + key := Key{Index: i, Constraint: constr.String()} + structLayouts[key] = &lyt + i++ + } + }) + return structLayouts, err +} + +func getLayoutForArch(v *semver.Version, arch string) (Key, *Layout, error) { + entries, err := generatedLayouts.ReadDir(filepath.Join(layoutDir, arch)) + if err != nil { + return Key{}, nil, err + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + var data []byte + data, err = generatedLayouts.ReadFile(filepath.Join(layoutDir, arch, entry.Name())) + if err != nil { + return Key{}, nil, err + } + ext := filepath.Ext(entry.Name()) + // Filter out non-yaml files. + if ext != ".yaml" && ext != ".yml" { + continue + } + var lyt Layout + if err = yaml.Unmarshal(data, &lyt); err != nil { + return Key{}, nil, err + } + rawConstraint := strings.TrimSuffix(entry.Name(), ext) + constr, err := semver.NewConstraint(rawConstraint) + if err != nil { + return Key{}, nil, err + } + key := Key{Constraint: constr.String()} + if constr.Check(v) { + return key, &lyt, nil + } + } + return Key{}, nil, errors.New("not found") +} + +// GetLayout returns the layout for the given version. +func GetLayout(v *semver.Version) (Key, *Layout, error) { + for k, l := range structLayouts { + constr, err := semver.NewConstraint(k.Constraint) + if err != nil { + return k, nil, err + } + if constr.Check(v) { + return k, l, nil + } + } + return Key{}, nil, errors.New("not found") +} + +// GetLayouts returns all the layouts. +func GetLayouts() (map[Key]*Layout, error) { + return structLayouts, nil +} diff --git a/pkg/libc/musl/musl_test.go b/pkg/libc/musl/musl_test.go new file mode 100644 index 0000000..c4fcced --- /dev/null +++ b/pkg/libc/musl/musl_test.go @@ -0,0 +1,68 @@ +package musl + +import ( + "testing" + + "github.com/Masterminds/semver/v3" + "github.com/google/go-cmp/cmp" +) + +func Test_getLayoutForArch(t *testing.T) { + + tests := []struct { + name string + v *semver.Version + arch string + want *Layout + wantErr bool + }{ + { + name: "1.2.2", + v: semver.MustParse("1.2.2"), + arch: "amd64", + want: &Layout{ + PthreadSize: 200, + PthreadTSD: 128, + }, + }, + { + name: "1.2.2", + v: semver.MustParse("1.2.2"), + arch: "arm64", + want: &Layout{ + PthreadSize: 200, + PthreadTSD: 112, + }, + }, + { + name: "1.1.19", + v: semver.MustParse("1.1.19"), + arch: "amd64", + want: &Layout{ + PthreadSize: 280, + PthreadTSD: 152, + }, + }, + { + name: "1.1.19", + v: semver.MustParse("1.1.19"), + arch: "arm64", + want: &Layout{ + PthreadSize: 280, + PthreadTSD: 152, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, got, err := getLayoutForArch(tt.v, tt.arch) + if (err != nil) != tt.wantErr { + t.Errorf("getLayoutForArch(%s) on %s error = %v, wantErr %v", tt.name, tt.arch, err, tt.wantErr) + return + } + if diff := cmp.Diff(tt.want, got, cmp.AllowUnexported(Layout{})); diff != "" { + t.Errorf("getLayoutForArch(%s) on %s mismatch (-want +got):\n%s", tt.name, tt.arch, diff) + } + }) + } +} diff --git a/scripts/download/musl.sh b/scripts/download/musl.sh new file mode 100755 index 0000000..a941ea4 --- /dev/null +++ b/scripts/download/musl.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +# Copyright 2022-2024 The Parca Authors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +TEMP_DIR=${TEMP_DIR:-tmp} +PACKAGE_DIR=${PACKAGE_DIR:-${TEMP_DIR}/apk} +BIN_DIR=${BIN_DIR:-${TEMP_DIR}/bin} +PACKAGE_NAME=${PACKAGE_NAME:-musl} +DEBUGINFO_DIR=${DEBUGINFO_DIR:-${TEMP_DIR}/debuginfo} + +# https://dl-cdn.alpinelinux.org/alpine/ +alpine_versions=( + v3.0 + v3.1 + v3.2 + v3.3 + v3.4 + v3.5 + v3.6 + v3.7 + v3.8 + v3.9 + v3.10 + v3.11 + v3.12 + v3.13 + v3.14 + v3.15 + v3.16 + v3.17 + v3.18 + v3.19 +) + +convertArch() { + case $1 in + amd64) + echo "x86_64" + ;; + arm64) + echo "aarch64" + ;; + esac +} + +echo "Downloading alpine runtimes" +for version in "${alpine_versions[@]}"; do + ./apkdownload -u "https://dl-cdn.alpinelinux.org/alpine/${version}/main" -t "${PACKAGE_DIR}" -o "${BIN_DIR}" -p "${PACKAGE_NAME}" +done + +echo "Extracting debuginfo from $BIN_DIR/$PACKAGE_NAME" +for arch in $BIN_DIR/$PACKAGE_NAME/*; do + for version in $arch/*; do + for variant in $version/*; do + if [ -d "$variant" ]; then + a=$(basename "$arch") + v=$(basename "$version") + linuxArch=$(convertArch "$(basename "$arch")") + if [ $(basename "$variant") == "dbg" ]; then + dbginfo="$variant"/usr/lib/debug/lib/ld-musl-"$linuxArch".so.1.debug + if [ -f "$dbginfo" ]; then + echo "copying $dbginfo to $DEBUGINFO_DIR/$PACKAGE_NAME/$a/$v/" + dbgTarget="$DEBUGINFO_DIR"/$PACKAGE_NAME/$a/$v/ + mkdir -p "$dbgTarget" + cp "$dbginfo" "$dbgTarget" + continue + fi + fi + fi + done + done +done diff --git a/scripts/mergelayout/musl.sh b/scripts/mergelayout/musl.sh new file mode 100755 index 0000000..990a6b1 --- /dev/null +++ b/scripts/mergelayout/musl.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Copyright 2022-2024 The Parca Authors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +TEMP_DIR=${TEMP_DIR:-tmp} +ARCH=${ARCH:-""} + +target_archs=( + amd64 + arm64 +) +if [ -n "${ARCH}" ]; then + target_archs=("${ARCH}") +fi + +TARGET_DIR=${TARGET_DIR:-pkg/libc/musl/layout} +rm -rf "${TARGET_DIR}" || true +mkdir -p "${TARGET_DIR}" +for arch in "${target_archs[@]}"; do + mkdir -p "${TARGET_DIR}/${arch}" + ./mergelayout -o "${TARGET_DIR}/${arch}" ${TEMP_DIR}/musl/${arch}/layout/'musl_*.yaml' +done diff --git a/scripts/structlayout/musl.sh b/scripts/structlayout/musl.sh new file mode 100755 index 0000000..8829ff6 --- /dev/null +++ b/scripts/structlayout/musl.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Copyright 2022-2024 The Parca Authors +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +TEMP_DIR=${TEMP_DIR:-tmp} +SOURCE_DIR=${SOURCE_DIR:-${TEMP_DIR}/debuginfo/musl} +TARGET_DIR=${TARGET_DIR:-${TEMP_DIR}/musl} + +mkdir -p "${TARGET_DIR}" +for arch in "${SOURCE_DIR}"/*; do + for version in "${arch}"/*; do + for dbgfile in "${version}"/*; do + v=$(basename "${version}") + a=$(basename "${arch}") + echo "Running structlayout against musl ${v} for ${a}..." + ./structlayout -r musl -v "${v}" -o "${TARGET_DIR}/${a}" "${dbgfile}" + done + done +done