From 2ad91df41e7839c7932bde1771a3024559add1c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Sun, 10 Nov 2024 15:15:14 +0100 Subject: [PATCH 1/9] README: fix path for coredump test suite --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f3c1bd1..2ab16aa7 100644 --- a/README.md +++ b/README.md @@ -476,7 +476,7 @@ The host agent code is tested with three test suites: tests. This works great for the user-land portion of the agent, but is unable to test any of the unwinding logic and BPF interaction. - **coredump test suite**\ - The coredump test suite (`utils/coredump`) we compile the whole BPF unwinder + The coredump test suite (`tools/coredump`) we compile the whole BPF unwinder code into a user-mode executable, then use the information from a coredump to simulate a realistic environment to test the unwinder code in. The coredump suite essentially implements all required BPF helper functions in user-space, From 0177cefe8a75b14b007e8b11d9d6e1452bbd09eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Sun, 10 Nov 2024 15:37:54 +0100 Subject: [PATCH 2/9] coredump: Add gosym sub command gosym symbolizes go test cases using addr2line. This is useful for debugging the correctness of go stack unwinding results. --- tools/coredump/coredump.go | 38 ++++++- tools/coredump/gosym.go | 206 +++++++++++++++++++++++++++++++++++++ tools/coredump/json.go | 9 +- tools/coredump/main.go | 1 + 4 files changed, 248 insertions(+), 6 deletions(-) create mode 100644 tools/coredump/gosym.go diff --git a/tools/coredump/coredump.go b/tools/coredump/coredump.go index eff7b37c..c3506d79 100644 --- a/tools/coredump/coredump.go +++ b/tools/coredump/coredump.go @@ -10,6 +10,8 @@ import ( "fmt" "os" "runtime" + "strconv" + "strings" "time" "unsafe" @@ -114,16 +116,44 @@ func (c *symbolizationCache) symbolize(ty libpf.FrameType, fileID libpf.FileID, } if data, ok := c.symbols[libpf.NewFrameID(fileID, lineNumber)]; ok { - return fmt.Sprintf("%s+%d in %s:%d", - data.FunctionName, data.FunctionOffset, - data.SourceFile, data.SourceLine), nil + return formatSymbolizedFrame(data, true), nil } sourceFile, ok := c.files[fileID] if !ok { sourceFile = fmt.Sprintf("%08x", fileID) } - return fmt.Sprintf("%s+0x%x", sourceFile, lineNumber), nil + return formatUnsymbolizedFrame(sourceFile, lineNumber), nil +} + +func formatSymbolizedFrame(frame *reporter.FrameMetadataArgs, functionOffsets bool) string { + var funcOffset string + if functionOffsets { + funcOffset = "+" + strconv.Itoa(int(frame.FunctionOffset)) + } + return fmt.Sprintf("%s%s in %s:%d", + frame.FunctionName, funcOffset, + frame.SourceFile, frame.SourceLine) +} + +func formatUnsymbolizedFrame(file string, addr libpf.AddressOrLineno) string { + return fmt.Sprintf("%s+0x%x", file, addr) +} + +func parseUnsymbolizedFrame(frame string) (file string, addr libpf.AddressOrLineno, err error) { + fileS, addrS, found := strings.Cut(frame, "+0x") + if !found { + err = fmt.Errorf("bad frame string: %q", frame) + return + } + file = fileS + var addrU uint64 + addrU, err = strconv.ParseUint(addrS, 16, 64) + if err != nil { + return + } + addr = libpf.AddressOrLineno(addrU) + return } func ExtractTraces(ctx context.Context, pr process.Process, debug bool, diff --git a/tools/coredump/gosym.go b/tools/coredump/gosym.go new file mode 100644 index 00000000..f87e76c7 --- /dev/null +++ b/tools/coredump/gosym.go @@ -0,0 +1,206 @@ +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "strconv" + "strings" + + "github.com/peterbourgon/ff/v3/ffcli" + "go.opentelemetry.io/ebpf-profiler/libpf" + "go.opentelemetry.io/ebpf-profiler/reporter" + "go.opentelemetry.io/ebpf-profiler/tools/coredump/modulestore" +) + +type gosymCmd struct { + store *modulestore.Store + casePath string +} + +func newGosymCmd(store *modulestore.Store) *ffcli.Command { + args := &gosymCmd{store: store} + + set := flag.NewFlagSet("gosym", flag.ExitOnError) + set.StringVar(&args.casePath, "case", "", "Path of the test case to debug") + + return &ffcli.Command{ + Name: "gosym", + Exec: args.exec, + ShortUsage: "gosym", + ShortHelp: "Symbolize go test case", + FlagSet: set, + } +} + +func (cmd *gosymCmd) exec(context.Context, []string) (err error) { + // Validate arguments. + if cmd.casePath == "" { + return errors.New("please specify `-case`") + } + + var test *CoredumpTestCase + test, err = readTestCase(cmd.casePath) + if err != nil { + return fmt.Errorf("failed to read test case: %w", err) + } + + if got := len(test.Modules); got != 1 { + return fmt.Errorf("got=%d module but only 1 module is supported right now", got) + } + + binary, err := extractModuleToTempFile(cmd.store, test.Modules[0]) + if err != nil { + return fmt.Errorf("failed to extract binary: %w", err) + } + defer os.Remove(binary) + + addrs := map[libpf.AddressOrLineno]struct{}{} + frames := map[libpf.AddressOrLineno][]*string{} + for _, thread := range test.Threads { + for i, frame := range thread.Frames { + _, addr, err := parseUnsymbolizedFrame(frame) + if err != nil { + continue + } + addrs[addr] = struct{}{} + frames[addr] = append(frames[addr], &thread.Frames[i]) + } + } + + locs, err := goSymbolize(binary, addrs) + if err != nil { + return fmt.Errorf("failed to symbolize: %w", err) + } + + for addr, frame := range locs { + for _, frameS := range frames[addr] { + *frameS = formatSymbolizedFrame(frame, false) + " (" + *frameS + ")" + } + } + + return writeTestCaseJSON(os.Stdout, test) +} + +func extractModuleToTempFile(store *modulestore.Store, m ModuleInfo) (string, error) { + file, err := os.CreateTemp("", "") + if err != nil { + return "", err + } + return file.Name(), store.UnpackModuleToPath(m.Ref, file.Name()) +} + +func goSymbolize(binary string, addrs map[libpf.AddressOrLineno]struct{}) (map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs, error) { + // Launch addr2line process. + addr2line := exec.Command("go", "tool", "addr2line", binary) + inR, inW := io.Pipe() + outR, outW := io.Pipe() + addr2line.Stdin = inR + addr2line.Stdout = outW + addr2line.Stderr = outW + if err := addr2line.Start(); err != nil { + return nil, err + } + + // Transform addrs into a list. This allows us to figure out which addr2line + // output corresponds to which address. + addrList := make([]libpf.AddressOrLineno, 0, len(addrs)) + for pc, _ := range addrs { + addrList = append(addrList, pc) + } + + // Parse addr2line output and map it to addrs we were given. + frames := map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs{} + scanCh := make(chan error) + go func() { + // Drain the output pipe in case we hit a parsing error. + defer io.Copy(io.Discard, outR) + + var err error + readFrame := addr2LineFrameReader(outR) + for { + var frame *reporter.FrameMetadataArgs + frame, err = readFrame() + if err != nil { + break + } + addr := addrList[len(frames)] + frames[addr] = frame + } + scanCh <- err + }() + + // Write addrList to addr2line stdin. + var writeErr error + writeAddr := addr2LineAddrWriter(inW) + for _, addr := range addrList { + if writeErr = writeAddr(addr); writeErr != nil { + break + } + } + + // Close the input pipe to signal addr2line that we're done. + if err := inW.Close(); err != nil { + return nil, err + // Wait for addr2line to finish. + } else if err := addr2line.Wait(); err != nil { + return nil, err + // Signal the output reader that we're done. + } else if err := outW.Close(); err != nil { + return nil, err + // Wait for the output reader to finish. + } else if err := <-scanCh; err != nil && err != io.EOF { + return nil, err + } + return frames, writeErr +} + +func addr2LineAddrWriter(w io.Writer) func(libpf.AddressOrLineno) error { + return func(addr libpf.AddressOrLineno) error { + _, err := fmt.Fprintf(w, "%x\n", addr) + return err + } +} + +func addr2LineFrameReader(r io.Reader) func() (*reporter.FrameMetadataArgs, error) { + scanner := bufio.NewScanner(r) + scanErr := func() error { + if err := scanner.Err(); err != nil { + return err + } + return io.EOF + } + var pair [2]string + return func() (*reporter.FrameMetadataArgs, error) { + if !scanner.Scan() { + return nil, scanErr() + } + pair[0] = scanner.Text() + if !scanner.Scan() { + return nil, fmt.Errorf("expected second line, but got: %w", scanErr()) + } + pair[1] = scanner.Text() + return linePairToFrame(pair) + } +} + +func linePairToFrame(pair [2]string) (*reporter.FrameMetadataArgs, error) { + var frame reporter.FrameMetadataArgs + frame.FunctionName = pair[0] + file, line, found := strings.Cut(pair[1], ":") + if !found { + return nil, fmt.Errorf("expected file:line but got: %q", pair[1]) + } + lineNum, err := strconv.Atoi(line) + if err != nil { + return nil, fmt.Errorf("invalid line number: %q", line) + } + frame.SourceFile = file + frame.SourceLine = libpf.SourceLineno(lineNum) + return &frame, nil +} diff --git a/tools/coredump/json.go b/tools/coredump/json.go index 388691de..4f904387 100644 --- a/tools/coredump/json.go +++ b/tools/coredump/json.go @@ -8,6 +8,7 @@ package main import ( "encoding/json" "fmt" + "io" "os" "path/filepath" "runtime" @@ -73,13 +74,17 @@ func writeTestCase(path string, c *CoredumpTestCase, allowOverwrite bool) error return fmt.Errorf("failed to create JSON file: %w", err) } - enc := json.NewEncoder(jsonFile) + return writeTestCaseJSON(jsonFile, c) +} + +// writeTestCaseJSON writes a test case to the given writer as JSON. +func writeTestCaseJSON(w io.Writer, c *CoredumpTestCase) error { + enc := json.NewEncoder(w) enc.SetIndent("", " ") enc.SetEscapeHTML(false) if err := enc.Encode(c); err != nil { return fmt.Errorf("JSON Marshall failed: %w", err) } - return nil } diff --git a/tools/coredump/main.go b/tools/coredump/main.go index d24aeb37..9be2501a 100644 --- a/tools/coredump/main.go +++ b/tools/coredump/main.go @@ -58,6 +58,7 @@ func main() { newRebaseCmd(store), newUploadCmd(store), newGdbCmd(store), + newGosymCmd(store), }, Exec: func(context.Context, []string) error { return flag.ErrHelp From 557e1509eba8c2f38e9e59c8ab5e64e6710a88dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Sun, 10 Nov 2024 11:16:36 -0500 Subject: [PATCH 3/9] coredump/gosym: support test cases with multiple modules This enables gosym to work with tesdata/arm64/go.symbhack.readheader.json which references several modules. --- tools/coredump/gosym.go | 81 ++++++++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 21 deletions(-) diff --git a/tools/coredump/gosym.go b/tools/coredump/gosym.go index f87e76c7..cc42081a 100644 --- a/tools/coredump/gosym.go +++ b/tools/coredump/gosym.go @@ -9,6 +9,7 @@ import ( "io" "os" "os/exec" + "path/filepath" "strconv" "strings" @@ -50,52 +51,90 @@ func (cmd *gosymCmd) exec(context.Context, []string) (err error) { return fmt.Errorf("failed to read test case: %w", err) } - if got := len(test.Modules); got != 1 { - return fmt.Errorf("got=%d module but only 1 module is supported right now", got) + module, addrs, err := goModuleAddrs(test) + if err != nil { + return fmt.Errorf("failed to find go module addresses: %w", err) } - binary, err := extractModuleToTempFile(cmd.store, test.Modules[0]) + binary, err := extractModuleToTempFile(cmd.store, module) if err != nil { return fmt.Errorf("failed to extract binary: %w", err) } defer os.Remove(binary) - addrs := map[libpf.AddressOrLineno]struct{}{} - frames := map[libpf.AddressOrLineno][]*string{} - for _, thread := range test.Threads { - for i, frame := range thread.Frames { - _, addr, err := parseUnsymbolizedFrame(frame) - if err != nil { - continue - } - addrs[addr] = struct{}{} - frames[addr] = append(frames[addr], &thread.Frames[i]) - } - } - locs, err := goSymbolize(binary, addrs) if err != nil { return fmt.Errorf("failed to symbolize: %w", err) } for addr, frame := range locs { - for _, frameS := range frames[addr] { - *frameS = formatSymbolizedFrame(frame, false) + " (" + *frameS + ")" + for _, originFrame := range addrs[addr] { + *originFrame = formatSymbolizedFrame(frame, false) + " (" + *originFrame + ")" } } return writeTestCaseJSON(os.Stdout, test) } -func extractModuleToTempFile(store *modulestore.Store, m ModuleInfo) (string, error) { +// goModuleAddrs returns the go module and the addresses to symbolize for it +// mapped to pointers to the frames in c that reference them. +func goModuleAddrs(c *CoredumpTestCase) (*ModuleInfo, map[libpf.AddressOrLineno][]*string, error) { + type moduleAddrs struct { + module *ModuleInfo + addrs map[libpf.AddressOrLineno][]*string + } + + moduleNames := map[string]*moduleAddrs{} + for i, module := range c.Modules { + moduleName := filepath.Base(module.LocalPath) + if _, ok := moduleNames[moduleName]; ok { + return nil, nil, fmt.Errorf("ambiguous module name: %q", moduleName) + } + moduleNames[moduleName] = &moduleAddrs{ + module: &c.Modules[i], + addrs: map[libpf.AddressOrLineno][]*string{}, + } + } + + // maxAddrs is the module with the most addresses to symbolize. We use this + // as a heuristic to determine which module is the Go module we're + // interested in. + // TODO(fg) alternatively we could extract all modules and run some check on + // them to see if they are go binaries. But this is more complex, so the + // current heuristic should be good enough for now. + var maxAddrs *moduleAddrs + for _, thread := range c.Threads { + for i, frame := range thread.Frames { + moduleName, addr, err := parseUnsymbolizedFrame(frame) + if err != nil { + continue + } + + moduleAddrs, ok := moduleNames[moduleName] + if !ok { + return nil, nil, fmt.Errorf("module not found: %q", moduleName) + } + + moduleAddrs.addrs[addr] = append(moduleAddrs.addrs[addr], &thread.Frames[i]) + if maxAddrs == nil || len(moduleAddrs.addrs[addr]) > len(maxAddrs.addrs[addr]) { + maxAddrs = moduleAddrs + } + } + } + return maxAddrs.module, maxAddrs.addrs, nil +} + +func extractModuleToTempFile(store *modulestore.Store, m *ModuleInfo) (string, error) { file, err := os.CreateTemp("", "") if err != nil { return "", err } - return file.Name(), store.UnpackModuleToPath(m.Ref, file.Name()) + return file.Name(), store.UnpackModule(m.Ref, file) } -func goSymbolize(binary string, addrs map[libpf.AddressOrLineno]struct{}) (map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs, error) { +type addrSet[T any] map[libpf.AddressOrLineno]T + +func goSymbolize[T any](binary string, addrs addrSet[T]) (map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs, error) { // Launch addr2line process. addr2line := exec.Command("go", "tool", "addr2line", binary) inR, inW := io.Pipe() From 6e99bdd48398aef579fffbec92bc0129b942699e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Sun, 10 Nov 2024 18:47:53 +0100 Subject: [PATCH 4/9] coredump/gosym: use debug/gosym instead of addr2line --- tools/coredump/gosym.go | 139 ++++++++-------------------------------- 1 file changed, 28 insertions(+), 111 deletions(-) diff --git a/tools/coredump/gosym.go b/tools/coredump/gosym.go index cc42081a..b0cbcb44 100644 --- a/tools/coredump/gosym.go +++ b/tools/coredump/gosym.go @@ -1,17 +1,15 @@ package main import ( - "bufio" "context" + "debug/elf" + "debug/gosym" "errors" "flag" "fmt" "io" "os" - "os/exec" "path/filepath" - "strconv" - "strings" "github.com/peterbourgon/ff/v3/ffcli" "go.opentelemetry.io/ebpf-profiler/libpf" @@ -56,13 +54,13 @@ func (cmd *gosymCmd) exec(context.Context, []string) (err error) { return fmt.Errorf("failed to find go module addresses: %w", err) } - binary, err := extractModuleToTempFile(cmd.store, module) + goBinary, err := cmd.store.OpenReadAt(module.Ref) if err != nil { - return fmt.Errorf("failed to extract binary: %w", err) + return fmt.Errorf("failed to open module: %w", err) } - defer os.Remove(binary) + defer goBinary.Close() - locs, err := goSymbolize(binary, addrs) + locs, err := goSymbolize(goBinary, addrs) if err != nil { return fmt.Errorf("failed to symbolize: %w", err) } @@ -124,122 +122,41 @@ func goModuleAddrs(c *CoredumpTestCase) (*ModuleInfo, map[libpf.AddressOrLineno] return maxAddrs.module, maxAddrs.addrs, nil } -func extractModuleToTempFile(store *modulestore.Store, m *ModuleInfo) (string, error) { - file, err := os.CreateTemp("", "") - if err != nil { - return "", err - } - return file.Name(), store.UnpackModule(m.Ref, file) -} - type addrSet[T any] map[libpf.AddressOrLineno]T -func goSymbolize[T any](binary string, addrs addrSet[T]) (map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs, error) { - // Launch addr2line process. - addr2line := exec.Command("go", "tool", "addr2line", binary) - inR, inW := io.Pipe() - outR, outW := io.Pipe() - addr2line.Stdin = inR - addr2line.Stdout = outW - addr2line.Stderr = outW - if err := addr2line.Start(); err != nil { +func goSymbolize[T any](goBinary io.ReaderAt, addrs addrSet[T]) (map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs, error) { + exe, err := elf.NewFile(goBinary) + if err != nil { return nil, err } - // Transform addrs into a list. This allows us to figure out which addr2line - // output corresponds to which address. - addrList := make([]libpf.AddressOrLineno, 0, len(addrs)) - for pc, _ := range addrs { - addrList = append(addrList, pc) + lineTableData, err := exe.Section(".gopclntab").Data() + if err != nil { + return nil, err } - - // Parse addr2line output and map it to addrs we were given. - frames := map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs{} - scanCh := make(chan error) - go func() { - // Drain the output pipe in case we hit a parsing error. - defer io.Copy(io.Discard, outR) - - var err error - readFrame := addr2LineFrameReader(outR) - for { - var frame *reporter.FrameMetadataArgs - frame, err = readFrame() - if err != nil { - break - } - addr := addrList[len(frames)] - frames[addr] = frame - } - scanCh <- err - }() - - // Write addrList to addr2line stdin. - var writeErr error - writeAddr := addr2LineAddrWriter(inW) - for _, addr := range addrList { - if writeErr = writeAddr(addr); writeErr != nil { - break - } + lineTable := gosym.NewLineTable(lineTableData, exe.Section(".text").Addr) + if err != nil { + return nil, err } - // Close the input pipe to signal addr2line that we're done. - if err := inW.Close(); err != nil { - return nil, err - // Wait for addr2line to finish. - } else if err := addr2line.Wait(); err != nil { - return nil, err - // Signal the output reader that we're done. - } else if err := outW.Close(); err != nil { - return nil, err - // Wait for the output reader to finish. - } else if err := <-scanCh; err != nil && err != io.EOF { + symTableData, err := exe.Section(".gosymtab").Data() + if err != nil { return nil, err } - return frames, writeErr -} -func addr2LineAddrWriter(w io.Writer) func(libpf.AddressOrLineno) error { - return func(addr libpf.AddressOrLineno) error { - _, err := fmt.Fprintf(w, "%x\n", addr) - return err + symTable, err := gosym.NewTable(symTableData, lineTable) + if err != nil { + return nil, err } -} -func addr2LineFrameReader(r io.Reader) func() (*reporter.FrameMetadataArgs, error) { - scanner := bufio.NewScanner(r) - scanErr := func() error { - if err := scanner.Err(); err != nil { - return err - } - return io.EOF - } - var pair [2]string - return func() (*reporter.FrameMetadataArgs, error) { - if !scanner.Scan() { - return nil, scanErr() - } - pair[0] = scanner.Text() - if !scanner.Scan() { - return nil, fmt.Errorf("expected second line, but got: %w", scanErr()) + frames := map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs{} + for addr, _ := range addrs { + file, line, fn := symTable.PCToLine(uint64(addr)) + frames[addr] = &reporter.FrameMetadataArgs{ + FunctionName: fn.Name, + SourceFile: file, + SourceLine: libpf.SourceLineno(line), } - pair[1] = scanner.Text() - return linePairToFrame(pair) - } -} - -func linePairToFrame(pair [2]string) (*reporter.FrameMetadataArgs, error) { - var frame reporter.FrameMetadataArgs - frame.FunctionName = pair[0] - file, line, found := strings.Cut(pair[1], ":") - if !found { - return nil, fmt.Errorf("expected file:line but got: %q", pair[1]) - } - lineNum, err := strconv.Atoi(line) - if err != nil { - return nil, fmt.Errorf("invalid line number: %q", line) } - frame.SourceFile = file - frame.SourceLine = libpf.SourceLineno(lineNum) - return &frame, nil + return frames, nil } From f9bc8ea6147e7568ed9faec601772025fde1f4ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Sun, 10 Nov 2024 21:35:31 +0100 Subject: [PATCH 5/9] coredump/gosym: better go module detection --- tools/coredump/gosym.go | 119 ++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 71 deletions(-) diff --git a/tools/coredump/gosym.go b/tools/coredump/gosym.go index b0cbcb44..26d1fb90 100644 --- a/tools/coredump/gosym.go +++ b/tools/coredump/gosym.go @@ -7,7 +7,6 @@ import ( "errors" "flag" "fmt" - "io" "os" "path/filepath" @@ -49,114 +48,92 @@ func (cmd *gosymCmd) exec(context.Context, []string) (err error) { return fmt.Errorf("failed to read test case: %w", err) } - module, addrs, err := goModuleAddrs(test) + symTable, addrs, err := goModuleAddrs(cmd.store, test) if err != nil { return fmt.Errorf("failed to find go module addresses: %w", err) } - goBinary, err := cmd.store.OpenReadAt(module.Ref) - if err != nil { - return fmt.Errorf("failed to open module: %w", err) - } - defer goBinary.Close() - - locs, err := goSymbolize(goBinary, addrs) - if err != nil { - return fmt.Errorf("failed to symbolize: %w", err) - } - - for addr, frame := range locs { - for _, originFrame := range addrs[addr] { - *originFrame = formatSymbolizedFrame(frame, false) + " (" + *originFrame + ")" + for addr, originFrames := range addrs { + file, line, fn := symTable.PCToLine(uint64(addr)) + for _, originFrame := range originFrames { + frame := reporter.FrameMetadataArgs{ + FunctionName: fn.Name, + SourceFile: file, + SourceLine: libpf.SourceLineno(line), + } + *originFrame = formatSymbolizedFrame(&frame, false) + " (" + *originFrame + ")" } } return writeTestCaseJSON(os.Stdout, test) } -// goModuleAddrs returns the go module and the addresses to symbolize for it -// mapped to pointers to the frames in c that reference them. -func goModuleAddrs(c *CoredumpTestCase) (*ModuleInfo, map[libpf.AddressOrLineno][]*string, error) { - type moduleAddrs struct { - module *ModuleInfo - addrs map[libpf.AddressOrLineno][]*string - } - - moduleNames := map[string]*moduleAddrs{} - for i, module := range c.Modules { - moduleName := filepath.Base(module.LocalPath) - if _, ok := moduleNames[moduleName]; ok { - return nil, nil, fmt.Errorf("ambiguous module name: %q", moduleName) - } - moduleNames[moduleName] = &moduleAddrs{ - module: &c.Modules[i], - addrs: map[libpf.AddressOrLineno][]*string{}, +// goModuleAddrs returns the symtable for the go module of test case and the +// addresses to symbolize for it mapped to pointers to the frames in the test +// case that reference them. +func goModuleAddrs(store *modulestore.Store, c *CoredumpTestCase) (*gosym.Table, map[libpf.AddressOrLineno][]*string, error) { + var symTable *gosym.Table + var module *ModuleInfo + for i := range c.Modules { + if table, err := gosymTable(store, &c.Modules[i]); err != nil { + continue + } else if symTable != nil { + return nil, nil, fmt.Errorf("multiple go modules found") + } else { + symTable = table + module = &c.Modules[i] } } - // maxAddrs is the module with the most addresses to symbolize. We use this - // as a heuristic to determine which module is the Go module we're - // interested in. - // TODO(fg) alternatively we could extract all modules and run some check on - // them to see if they are go binaries. But this is more complex, so the - // current heuristic should be good enough for now. - var maxAddrs *moduleAddrs + addrs := map[libpf.AddressOrLineno][]*string{} + moduleName := filepath.Base(module.LocalPath) for _, thread := range c.Threads { for i, frame := range thread.Frames { - moduleName, addr, err := parseUnsymbolizedFrame(frame) + frameModuleName, addr, err := parseUnsymbolizedFrame(frame) if err != nil { continue } - moduleAddrs, ok := moduleNames[moduleName] - if !ok { - return nil, nil, fmt.Errorf("module not found: %q", moduleName) + if frameModuleName != moduleName { + continue } - moduleAddrs.addrs[addr] = append(moduleAddrs.addrs[addr], &thread.Frames[i]) - if maxAddrs == nil || len(moduleAddrs.addrs[addr]) > len(maxAddrs.addrs[addr]) { - maxAddrs = moduleAddrs - } + addrs[addr] = append(addrs[addr], &thread.Frames[i]) } } - return maxAddrs.module, maxAddrs.addrs, nil + return symTable, addrs, nil } -type addrSet[T any] map[libpf.AddressOrLineno]T - -func goSymbolize[T any](goBinary io.ReaderAt, addrs addrSet[T]) (map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs, error) { - exe, err := elf.NewFile(goBinary) +func gosymTable(store *modulestore.Store, module *ModuleInfo) (*gosym.Table, error) { + reader, err := store.OpenReadAt(module.Ref) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to open module: %w", err) } + defer reader.Close() - lineTableData, err := exe.Section(".gopclntab").Data() + exe, err := elf.NewFile(reader) if err != nil { return nil, err } - lineTable := gosym.NewLineTable(lineTableData, exe.Section(".text").Addr) - if err != nil { - return nil, err + + textSection := exe.Section(".text") + if textSection == nil { + return nil, errors.New("missing .text section") } - symTableData, err := exe.Section(".gosymtab").Data() + pclntab := exe.Section(".gopclntab") + if pclntab == nil { + return nil, errors.New("missing .gopclntab section") + } + + lineTableData, err := pclntab.Data() if err != nil { return nil, err } - - symTable, err := gosym.NewTable(symTableData, lineTable) + lineTable := gosym.NewLineTable(lineTableData, textSection.Addr) if err != nil { return nil, err } - frames := map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs{} - for addr, _ := range addrs { - file, line, fn := symTable.PCToLine(uint64(addr)) - frames[addr] = &reporter.FrameMetadataArgs{ - FunctionName: fn.Name, - SourceFile: file, - SourceLine: libpf.SourceLineno(line), - } - } - return frames, nil + return gosym.NewTable(nil, lineTable) } From e540dd4906cf8a2fe78a6f028c574b2cca774129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Sun, 10 Nov 2024 15:42:09 -0500 Subject: [PATCH 6/9] fix golint suggestions --- tools/coredump/gosym.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tools/coredump/gosym.go b/tools/coredump/gosym.go index 26d1fb90..8ca5ae5c 100644 --- a/tools/coredump/gosym.go +++ b/tools/coredump/gosym.go @@ -75,14 +75,14 @@ func goModuleAddrs(store *modulestore.Store, c *CoredumpTestCase) (*gosym.Table, var symTable *gosym.Table var module *ModuleInfo for i := range c.Modules { - if table, err := gosymTable(store, &c.Modules[i]); err != nil { + table, err := gosymTable(store, &c.Modules[i]) + if err != nil { continue } else if symTable != nil { - return nil, nil, fmt.Errorf("multiple go modules found") - } else { - symTable = table - module = &c.Modules[i] + return nil, nil, errors.New("multiple go modules found") } + symTable = table + module = &c.Modules[i] } addrs := map[libpf.AddressOrLineno][]*string{} @@ -131,9 +131,5 @@ func gosymTable(store *modulestore.Store, module *ModuleInfo) (*gosym.Table, err return nil, err } lineTable := gosym.NewLineTable(lineTableData, textSection.Addr) - if err != nil { - return nil, err - } - return gosym.NewTable(nil, lineTable) } From 384b55b316b8f417a89f1733027b1bcce2ae2295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Sun, 10 Nov 2024 15:45:02 -0500 Subject: [PATCH 7/9] fix more golint --- tools/coredump/gosym.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/coredump/gosym.go b/tools/coredump/gosym.go index 8ca5ae5c..4207c607 100644 --- a/tools/coredump/gosym.go +++ b/tools/coredump/gosym.go @@ -71,7 +71,9 @@ func (cmd *gosymCmd) exec(context.Context, []string) (err error) { // goModuleAddrs returns the symtable for the go module of test case and the // addresses to symbolize for it mapped to pointers to the frames in the test // case that reference them. -func goModuleAddrs(store *modulestore.Store, c *CoredumpTestCase) (*gosym.Table, map[libpf.AddressOrLineno][]*string, error) { +func goModuleAddrs(store *modulestore.Store, c *CoredumpTestCase) ( + *gosym.Table, map[libpf.AddressOrLineno][]*string, error, +) { var symTable *gosym.Table var module *ModuleInfo for i := range c.Modules { From 794aa882ebcdb37f126a68ec03b22e30e6061249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Sun, 10 Nov 2024 19:17:30 -0500 Subject: [PATCH 8/9] coredump/gosym: add comment about locating .gocplntab --- tools/coredump/gosym.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/coredump/gosym.go b/tools/coredump/gosym.go index 4207c607..5cbfc8a8 100644 --- a/tools/coredump/gosym.go +++ b/tools/coredump/gosym.go @@ -123,6 +123,9 @@ func gosymTable(store *modulestore.Store, module *ModuleInfo) (*gosym.Table, err return nil, errors.New("missing .text section") } + // TODO(fg): The section headers might be stripped, in which case we could + // try to locate gopclntab using alternative heuristics. See + // nativeunwind/elfunwindinfo/elfgopclntab.go for code that does this. pclntab := exe.Section(".gopclntab") if pclntab == nil { return nil, errors.New("missing .gopclntab section") From efd40d30d0a4d08b4f9cf8bda7fd2abb41a5c75f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Mon, 11 Nov 2024 07:12:22 -0500 Subject: [PATCH 9/9] coredump/gosym: better .text section addr discovery --- tools/coredump/gosym.go | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/tools/coredump/gosym.go b/tools/coredump/gosym.go index 5cbfc8a8..110e1a8b 100644 --- a/tools/coredump/gosym.go +++ b/tools/coredump/gosym.go @@ -76,9 +76,11 @@ func goModuleAddrs(store *modulestore.Store, c *CoredumpTestCase) ( ) { var symTable *gosym.Table var module *ModuleInfo + var errs []error for i := range c.Modules { table, err := gosymTable(store, &c.Modules[i]) if err != nil { + errs = append(errs, err) continue } else if symTable != nil { return nil, nil, errors.New("multiple go modules found") @@ -87,6 +89,10 @@ func goModuleAddrs(store *modulestore.Store, c *CoredumpTestCase) ( module = &c.Modules[i] } + if module == nil { + return nil, nil, fmt.Errorf("no go module found: %w", errors.Join(errs...)) + } + addrs := map[libpf.AddressOrLineno][]*string{} moduleName := filepath.Base(module.LocalPath) for _, thread := range c.Threads { @@ -118,9 +124,25 @@ func gosymTable(store *modulestore.Store, module *ModuleInfo) (*gosym.Table, err return nil, err } - textSection := exe.Section(".text") - if textSection == nil { - return nil, errors.New("missing .text section") + // Look up the address of the .text section. + var textAddr uint64 + if sect := exe.Section(".text"); sect != nil { + textAddr = sect.Addr + } + + // But prefer the runtime.text symbol if it exists. This is modeled after go + // tool addr2line, see src/cmd/internal/objfile/objfile.go in the Go tree. + symbols, err := exe.Symbols() + if err == nil { + for _, sym := range symbols { + if sym.Name == "runtime.text" { + textAddr = sym.Value + } + } + } + + if textAddr == 0 { + return nil, errors.New("missing .text section and runtime.text symbol") } // TODO(fg): The section headers might be stripped, in which case we could @@ -135,6 +157,6 @@ func gosymTable(store *modulestore.Store, module *ModuleInfo) (*gosym.Table, err if err != nil { return nil, err } - lineTable := gosym.NewLineTable(lineTableData, textSection.Addr) + lineTable := gosym.NewLineTable(lineTableData, textAddr) return gosym.NewTable(nil, lineTable) }