chore(workflow): add release workflow etc. (#61)

* add release workflow.
* add more arch and vendor btf support.
* add complete btf files embed support.
* add commitizen integration.

.github/workflows/release-test.yml

@@ -3,7 +3,7 @@ name: release-test
 on:
   workflow_dispatch:
   push:
-    branches: [ "master", "dev", "feature/e2e" ]
+    branches: [ "master", "dev" ]
   pull_request:
     branches: [ "master", "dev" ]
 
@@ -15,13 +15,19 @@ jobs:
   release-test:
     runs-on: ubuntu-latest
     steps:
+
+      - name: Remove large directories
+        run: |
+          df -h
+          cd /opt/hostedtoolcache
+          find . -mindepth 1 -maxdepth 1 -type d -not -name 'go' -exec rm -rf {} +
+          df -h
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
 
       - name: Set up Go
         uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5
         with:
-          go-version: '1.22.4'
-
+          go-version: '1.22.6'
       - name: Run GoReleaser Draft
         uses: goreleaser/goreleaser-action@286f3b13b1b49da4ac219696163fb8c1c93e1200 # v6
         with:
@@ -43,7 +49,7 @@ jobs:
         uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4
         with:
           name: kyanos_v0.0.0-next_linux_amd64.tar.gz
-          path: dist/kyanos.0.0-next_linux_amd64.tar.gz
+          path: dist/kyanos_v0.0.0-next_linux_amd64.tar.gz
 
       - name: Store Releases
         uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4

.github/workflows/release.yml

@@ -0,0 +1,38 @@
+name: release
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - v*
+
+permissions:
+  contents: read
+
+jobs:
+
+  release-test:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Remove large directories
+        run: |
+          df -h
+          cd /opt/hostedtoolcache
+          find . -mindepth 1 -maxdepth 1 -type d -not -name 'go' -exec rm -rf {} +
+          df -h
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+
+      - name: Set up Go
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5
+        with:
+          go-version: '1.22.6'
+
+      - name: Run GoReleaser Draft
+        uses: goreleaser/goreleaser-action@286f3b13b1b49da4ac219696163fb8c1c93e1200 # v6
+        with:
+          distribution: goreleaser
+          version: latest
+          args: release --clean --draft=true -p 1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GORELEASE_TOKEN }}

.gitignore

@@ -9,6 +9,7 @@
 *.dylib
 *.o
 __debug**
+node_modules/
 
 # Test binary, built with `go test -c`
 *.test

.goreleaser.yaml

@@ -14,7 +14,7 @@ before:
     - sudo add-apt-repository "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main"
     - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 15CF4D18AF4F7421
     - sudo apt update
-    - sudo apt-get install -y gcc flex bison make autoconf libelf-dev gcc-aarch64-linux-gnu libc6-dev-arm64-cross pkg-config llvm clang
+    - sudo apt-get install -y gcc flex bison make autoconf libelf-dev gcc-aarch64-linux-gnu libc6-dev-arm64-cross pkg-config llvm clang rsync
     - git submodule update --init --recursive
 
 builds:
@@ -35,14 +35,16 @@ builds:
     hooks:
       pre:
         - bash -c 'sudo make clean || true'
-        - bash -c 'sudo make build-bpf || true' 
+        - bash -c 'sudo make build-bpf' 
+        - bash -c 'sudo make btfgen BUILD_ARCH=arm64 ARCH_BPF_NAME=arm64' 
+
 
   - id: kyanos-amd64
     binary: kyanos
     env:
       - CGO_ENABLED=1
-      - CC=clang
-      - CGO_LDFLAGS="-Xlinker -rpath=. -static"
+      - CC=gcc
+      # - CGO_LDFLAGS="-Xlinker -rpath=. -static"
     flags:
       - -tags=static
     ldflags:
@@ -55,7 +57,8 @@ builds:
     hooks:
       pre:
         - bash -c 'sudo make clean || true'
-        - bash -c 'sudo make build-bpf || true'
+        - bash -c 'sudo make build-bpf'
+        - bash -c 'sudo make btfgen BUILD_ARCH=x86_64 ARCH_BPF_NAME=x86' 
 
 archives:
   - builds:

Makefile

@@ -72,7 +72,12 @@ build-bpf: $(LIBBPF_OBJ) $(wildcard bpf/*.[ch]) | $(OUTPUT)
 kyanos: $(GO_FILES)
 	$(call msg,BINARY,$@)
 	export CGO_LDFLAGS="-Xlinker -rpath=. -static" && go build
-
+
+
+.PHONY: btfgen
+btfgen:
+	./bpf/btfgen.sh $(BUILD_ARCH) $(ARCH_BPF_NAME)
+
 # delete failed targets
 .DELETE_ON_ERROR:
 

bpf/btfgen.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+BASEDIR=$(dirname "${0}")
+cd ${BASEDIR}/../
+BASEDIR=$(pwd)
+cd ${BASEDIR}
+
+BTFHUB_REPO="https://github.com/aquasecurity/btfhub.git"
+BTFHUB_ARCH_REPO="https://github.com/aquasecurity/btfhub-archive.git"
+
+
+KYANOS_BPF_CORE="${BASEDIR}/bpf/agent_$2_bpfel.o"
+
+BTFHUB_DIR="${BASEDIR}/deps/btfhub"
+BTFHUB_ARCH_DIR="${BASEDIR}/deps/btfhub-archive"
+
+ARCH=$(uname -m)
+
+case ${ARCH} in
+"x86_64")
+    ARCH="x86_64"
+    ;;
+"aarch64")
+    ARCH="arm64"
+    ;;
+*)
+    die "unsupported architecture"
+    ;;
+esac
+
+
+die() {
+    echo ${@}
+    exit 1
+}
+
+branch_clean() {
+    cd ${1} || die "could not change dirs"
+
+    # small sanity check
+    [ ! -d ./.git ] && die "$(basename $(pwd)) not a repo dir"
+
+    # git fetch -a || die "could not fetch ${1}" # make sure its updated
+    # git clean -fdX                             # clean leftovers
+    # git reset --hard                           # reset letfovers
+    # git checkout origin/main -b main-$$
+    # git branch -D main
+    # git branch -m main-$$ main # origin/main == main
+
+    cd ${BASEDIR}
+}
+
+CMDS="rsync git cp rm mv"
+for cmd in ${CMDS}; do
+    command -v $cmd 2>&1 >/dev/null || die "cmd ${cmd} not found"
+done
+[ ! -f ${KYANOS_BPF_CORE} ] && die "kyanos CO-RE obj not found: ${KYANOS_BPF_CORE}"
+
+[ ! -d ${BTFHUB_DIR} ] && git clone "${BTFHUB_REPO}" ${BTFHUB_DIR}
+[ ! -d ${BTFHUB_ARCH_DIR} ] && git clone --depth=1 "${BTFHUB_ARCH_REPO}" ${BTFHUB_ARCH_DIR}
+
+if [ -z ${SKIP_FETCH} ]; then
+    branch_clean ${BTFHUB_DIR}
+    branch_clean ${BTFHUB_ARCH_DIR}
+fi
+
+cd ${BTFHUB_DIR}
+
+
+# sync only supported kernels
+
+ARCH_EXCLUDE=$(printf "x86_64\naarch64\n" | grep -v $(uname -m) | xargs)
+rsync -avz \
+    ${BTFHUB_ARCH_DIR}/ \
+    --exclude=.git* \
+    --exclude=README.md \
+    --exclude=${ARCH_EXCLUDE} \
+    --include='*ubuntu*' \
+    --include='*centos*' \
+    --include='*debian*' \
+    ./archive/
+
+# generate tailored BTFs
+
+[ ! -d ${BASEDIR}/bpf/custom-archive ] && mkdir -p ${BASEDIR}/bpf/custom-archive
+rm -rf ${BASEDIR}/bpf/custom-archive/* || true
+
+[ ! -f ./tools/btfgen.sh ] && die "could not find btfgen.sh"
+./tools/btfgen.sh -a $1 -o ${KYANOS_BPF_CORE}
+
+# move tailored BTFs to dist
+mv ./custom-archive/* ${BASEDIR}/bpf/custom-archive

bpf/loader/loader.go

@@ -158,6 +158,54 @@ func LoadBPF(options ac.AgentOptions) (*BPF, error) {
 	return bf, nil
 }
 
+var osReleaseFiles = []string{
+	"/etc/os-release",
+	"/usr/lib/os-release",
+}
+
+type Release struct {
+	Id        string
+	VersionId string
+}
+
+func getRelease() (*Release, error) {
+	var errors []error
+	for _, path := range osReleaseFiles {
+		data, err := os.ReadFile(path)
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+
+		var release Release
+		for _, line := range strings.Split(string(data), "\n") {
+			line := strings.TrimSpace(line)
+			parts := strings.Split(line, "=")
+			if len(parts) < 2 {
+				continue
+			}
+			key, value := parts[0], parts[1]
+			key = strings.TrimSpace(key)
+			switch key {
+			case "ID":
+				release.Id = strings.TrimSpace(value)
+				break
+			case "VERSION_ID":
+				release.VersionId = strings.TrimSpace(value)
+				break
+			}
+		}
+		if release.Id != "" {
+			return &release, nil
+		}
+	}
+
+	if len(errors) != 0 {
+		return nil, fmt.Errorf("%v", errors)
+	}
+
+	return nil, fmt.Errorf("can't get release info from %v", osReleaseFiles)
+}
 func getBestMatchedBTFFile() ([]uint8, error) {
 	if bpf.IsKernelSupportHasBTF() {
 		return nil, nil
@@ -167,25 +215,15 @@ func getBestMatchedBTFFile() ([]uint8, error) {
 	si.GetSysInfo()
 	common.AgentLog.Debugf("[sys info] vendor: %s, os_arch: %s, kernel_arch: %s", si.OS.Vendor, si.OS.Architecture, si.Kernel.Architecture)
 
-	if si.OS.Vendor != "ubuntu" && si.OS.Vendor != "centos" {
-		common.AgentLog.Fatalf("Current only support centos and ubuntu, current is %s\n completed OS info is: %v", si.OS.Vendor, si.OS)
-	}
-	if si.OS.Architecture != "amd64" {
-		common.AgentLog.Fatal("Current only support amd64")
-	}
-	if si.Kernel.Architecture != "x86_64" {
-		common.AgentLog.Fatal("Current only support x86_64")
-	}
 
-	var btfFileDir string
-	btfFileDir += "custom-archive"
-	btfFileDir += "/" + si.OS.Vendor
-	if si.OS.Vendor == "centos" {
-		btfFileDir += "/" + si.OS.Release[:1]
-	} else {
-		btfFileDir += "/" + si.OS.Release[:5]
-	}
-	btfFileDir += "/" + si.Kernel.Architecture
+	osInfo, err := common.GetOSInfo()
+	osId := osInfo.GetOSReleaseFieldValue(common.OS_ID)
+	versionId := strings.Replace(osInfo.GetOSReleaseFieldValue(common.OS_VERSION_ID), "\"", "", -1)
+	kernelRelease := osInfo.GetOSReleaseFieldValue(common.OS_KERNEL_RELEASE)
+	arch := osInfo.GetOSReleaseFieldValue(common.OS_ARCH)
+
+
+	btfFileDir := fmt.Sprintf("custom-archive/%s/%s/%s/%s.btf", osId, versionId, arch, kernelRelease)
 	dir, err := bpf.BtfFiles.ReadDir(btfFileDir)
 	if err != nil {
 		common.AgentLog.Warnf("btf file not exists, path: %s", btfFileDir)