Skip to content

Commit d87e5c5

Browse files
yaokunzhangCopilot
yaokunzhang
and
Copilot
authored
Import the sensleak and mirchecker module. (#1450)
* Import the sensleak and mirchecker module. * Import the sensleak and mirchecker module. * Import the sensleak and mirchecker module. * Update extensions/rust/sensleak/src/model.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Yaokun Zhang - nju <76199015+yaokunzhang@users.noreply.github.com> * Update extensions/rust/mirchecker/src/model.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Yaokun Zhang - nju <76199015+yaokunzhang@users.noreply.github.com> * Update extensions/rust/mirchecker/src/model.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Yaokun Zhang - nju <76199015+yaokunzhang@users.noreply.github.com> --------- Signed-off-by: Yaokun Zhang - nju <76199015+yaokunzhang@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent 80e4901 commit d87e5c5

20 files changed

Lines changed: 1686 additions & 0 deletions

‎extensions/rust/Cargo.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ members = [
88
"common/search",
99
"common/tudriver", # tugraph client, deal with the quary from router
1010
"tugraph",
11+
"sensleak",
12+
"mirchecker",
1113
]
1214

1315
resolver = "2" # use resolver of version 2
@@ -22,5 +24,7 @@ tudriver = { path = "common/tudriver" }
2224
database = { path = "common/database" }
2325
entity = { path = "common/database/entity" }
2426
import_tugraph = {path = "tugraph"}
27+
sensleak_analysis = {path = "sensleak"}
28+
mirchecker_analysis = {path = "mirchecker"}
2529

2630
# third-party (所有第三方依赖, 按字母表排序, 均保留1个小数点)

‎extensions/rust/mirchecker/.env‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Kafka配置
2+
KAFKA_BROKER=172.17.0.1:30092
3+
KAFKA_TOPIC=ANALYSIS.demo.20250902
4+
KAFKA_CONSUMER_GROUP_ID=sensleak-group32
5+
6+
PG_HOST=172.17.0.1
7+
PG_PORT=31432
8+
PG_USER=mega
9+
PG_PASSWORD=mega
10+
PG_DB=cratespro
11+
12+
# 分析器配置
13+
ANALYZERS_CONFIG=./analyzers.json
14+
CLONE_DIR=/tmp/analysis-repos

‎extensions/rust/mirchecker/BUCK‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
load("@prelude//rust:cargo_package.bzl", "cargo")
2+
3+
filegroup(
4+
name = "analysis-mirchecker-0.1.0.crate",
5+
srcs = [
6+
"src/main.rs",
7+
"src/model.rs",
8+
],
9+
)
10+
11+
pkg_deps = [
12+
"//third-party:anyhow",
13+
"//third-party:chrono",
14+
"//third-party:clap",
15+
"//third-party:dotenvy",
16+
"//third-party:git2",
17+
"//third-party:rdkafka",
18+
"//third-party:serde",
19+
"//third-party:serde_json",
20+
"//third-party:tempfile",
21+
"//third-party:tokio",
22+
"//third-party:tokio-postgres",
23+
"//third-party:tokio-process",
24+
"//third-party:tracing",
25+
"//third-party:tracing-subscriber",
26+
"//third-party:url",
27+
]
28+
29+
cargo.rust_binary(
30+
name = "analysis_mirchecker",
31+
srcs = [":analysis-mirchecker-0.1.0.crate"],
32+
crate_root = "analysis-mirchecker-0.1.0.crate/src/main.rs",
33+
edition = "2024",
34+
deps = pkg_deps,
35+
visibility = ["PUBLIC"],
36+
)

‎extensions/rust/mirchecker/Cargo.toml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
[package]
2+
name = "mirchecker"
3+
version = "0.1.0"
4+
edition = "2021"
5+
6+
[dependencies]
7+
# Kafka
8+
rdkafka = { version = "0.38", features = ["cmake-build"] }
9+
10+
# 异步运行时
11+
tokio = { version = "1.47", features = ["full"] }
12+
13+
# 序列化
14+
serde = { version = "1.0", features = ["derive"] }
15+
serde_json = "1.0"
16+
17+
# Git 操作
18+
git2 = "0.20"
19+
20+
# 命令行工具
21+
clap = { version = "4.5", features = ["derive", "env"] }
22+
23+
# 日志
24+
tracing = "0.1"
25+
tracing-subscriber = "0.3"
26+
27+
# 错误处理
28+
anyhow = "1.0"
29+
30+
# 进程调用
31+
tokio-process = "0.2"
32+
33+
tokio-postgres = "0.7"
34+
35+
# 文件系统操作
36+
tempfile = "3.19"
37+
38+
# URL 处理
39+
url = "2.5.7"
40+
41+
dotenvy = "0.15"
42+
43+
chrono = { version = "0.4", features = ["serde"] }

‎extensions/rust/mirchecker/Dockerfile‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
FROM c9-ssl32-gh19
2+
3+
# Install tools and dependencies
4+
# MirChecker (Dockerfile): gmp-devel, mpfr-devel, ppl-devel, z3-devel
5+
RUN dnf update -y \
6+
&& dnf group install -y "Development Tools" \
7+
&& dnf install -y glibc-langpack-en sudo tzdata \
8+
&& dnf install -y gmp-devel mpfr-devel \
9+
&& dnf install -y epel-release \
10+
&& dnf install -y ppl-devel z3-devel
11+
12+
# Switch to user (user 'rust' with UID 1000 already created in base image)
13+
ARG USERNAME="rust"
14+
ARG USER_UID="1000"
15+
USER $USERNAME
16+
17+
# Install Rust with nightly-2025-01-10 toolchain and required components
18+
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain nightly-2025-01-10 -y
19+
ENV PATH="/home/$USERNAME/.cargo/bin:$PATH"
20+
# Install required components for MirChecker
21+
RUN rustup component add clippy rustfmt rustc-dev rust-src rust-std llvm-tools-preview --toolchain nightly-2025-01-10
22+
23+
# Create and set permissions for workdir directory
24+
USER root
25+
RUN mkdir -p /workdir && chown $USERNAME:$USERNAME /workdir
26+
USER $USERNAME
27+
28+
WORKDIR /workdir
29+
30+
# Copy artifacts for analysis-mirchecker
31+
COPY ./analysis_mirchecker ./analysis_mirchecker
32+
COPY ./.env ./.env
33+
34+
# Copy artifacts for MirChecker
35+
COPY ./cargo-mir-checker ./cargo-mir-checker
36+
COPY ./mir-checker ./mir-checker
37+
ENV LD_LIBRARY_PATH="/home/$USERNAME/.rustup/toolchains/nightly-2025-01-10-x86_64-unknown-linux-gnu/lib:/usr/local/lib64:/usr/lib64"
38+
39+
ENV TZ=Asia/Shanghai
40+
ENV LANG=en_US.UTF-8
41+
ENV LC_ALL=en_US.UTF-8
42+
43+
ENV RUST_BACKTRACE=1
44+
CMD ["./analysis_mirchecker"]

‎extensions/rust/mirchecker/deploy-analysis-mirchecker.sh‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
set -euxo pipefail
3+
4+
# source
5+
CODE_DIR=$(pwd)
6+
INFRA_DIR=/home/rust/workspace/crates-pro-infra
7+
# deployment
8+
NAMESPACE=crates-pro
9+
DEPLOYMENT=analysis-mirchecker
10+
KAFKA_HOST=172.17.0.1:30092
11+
KAFKA_CONSUMER_GROUP=mirchecker-group27
12+
# build
13+
BUILD_DIR=$(mktemp -d)
14+
TIMESTAMP=$(date +%Y%m%d-%H%M)
15+
IMAGE_NAME=localhost:30500/cratespro-analysis-mirchecker:local-$TIMESTAMP
16+
17+
### Preparation: Sync source directories
18+
rsync --delete --archive $CODE_DIR/ $INFRA_DIR/project/crates-pro/analysis_mirchecker --exclude="/.git" --exclude="/bin" --exclude="/target"
19+
20+
### Step 1: Compile, then copy artifacts to $BUILD_DIR
21+
cd $INFRA_DIR
22+
mkdir -p $BUILD_DIR/bin
23+
buck2 build //project/crates-pro/analysis_mirchecker:analysis_mirchecker --out $BUILD_DIR/analysis_mirchecker
24+
25+
cp $CODE_DIR/bin/cargo-mir-checker $BUILD_DIR/cargo-mir-checker
26+
cp $CODE_DIR/bin/mir-checker $BUILD_DIR/mir-checker
27+
28+
cp $CODE_DIR/.env $BUILD_DIR/.env
29+
cd $CODE_DIR
30+
31+
### Step 2: Build Docker images
32+
docker build -t $IMAGE_NAME -f $CODE_DIR/Dockerfile $BUILD_DIR
33+
34+
### Step 3: Push Docker images
35+
docker push $IMAGE_NAME
36+
37+
### Step 4: Stop current containers
38+
# Scale deployment to 0 replicas
39+
kubectl scale deployment $DEPLOYMENT -n $NAMESPACE --replicas=0
40+
41+
# Wait until all pods are terminated
42+
while kubectl get pods -n $NAMESPACE | grep $DEPLOYMENT > /dev/null; do
43+
sleep 5
44+
done
45+
46+
### Step 5: Set new images
47+
kubectl set image deployment/$DEPLOYMENT -n $NAMESPACE container-0=$IMAGE_NAME
48+
49+
# # Wait until all kafka consumers are removed
50+
# while docker run --rm -t bitnami/kafka -- kafka-consumer-groups.sh --bootstrap-server $KAFKA_HOST --group $KAFKA_CONSUMER_GROUP --describe | grep rdkafka > /dev/null; do
51+
# sleep 5
52+
# done
53+
54+
### Step 6: Run new containers
55+
# Scale deployment back to 1 replica
56+
kubectl scale deployment $DEPLOYMENT -n $NAMESPACE --replicas=1

‎extensions/rust/mirchecker/deployment.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: analysis-mirchecker
5+
namespace: crates-pro
6+
spec:
7+
replicas: 1
8+
selector:
9+
matchLabels:
10+
app: analysis-mirchecker
11+
template:
12+
metadata:
13+
labels:
14+
app: analysis-mirchecker
15+
spec:
16+
securityContext:
17+
runAsUser: 1000
18+
runAsGroup: 1000
19+
fsGroup: 1000
20+
containers:
21+
- name: container-0
22+
image: localhost:30500/cratespro-analysis-mirchecker:local-20250905-1433
23+
imagePullPolicy: IfNotPresent
24+
env:
25+
- name: KAFKA_BROKER
26+
value: "my-kafka.kafka.svc.cluster.local:9092"
27+
- name: PG_HOST
28+
value: "postgresql-test1.postgresql.svc.cluster.local"
29+
- name: PG_PORT
30+
value: "5432"
31+
- name: CLONE_DIR
32+
value: "/tmp/analysis-repos"
33+
volumeMounts:
34+
- mountPath: /tmp/analysis-repos
35+
name: clone-dir-volume
36+
volumes:
37+
- name: clone-dir-volume
38+
persistentVolumeClaim:
39+
claimName: analysis-clone-dir-pvc

0 commit comments

Comments
 (0)