Skip to content

Commit bdd93d0

Browse files
sd109sd109
authored
Merge pull request #3 from stackhpc/update/upstream
Sync in latest upstream changes
2 parents 5581bd7 + 6a59674 commit bdd93d0

File tree

676 files changed

+41530
-24469
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

676 files changed

+41530
-24469
lines changed

‎.github/workflows/release.yml

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
name: release
2+
3+
on:
4+
schedule:
5+
- cron: '0 13 * * *' # This schedule runs every 13:00:00Z(21:00:00+08:00)
6+
# The "create tags" trigger is specifically focused on the creation of new tags, while the "push tags" trigger is activated when tags are pushed, including both new tag creations and updates to existing tags.
7+
create:
8+
tags:
9+
- "v*.*.*" # normal release
10+
- "nightly" # the only one mutable tag
11+
12+
# https://docs.github.com/en/actions/using-jobs/using-concurrency
13+
concurrency:
14+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
release:
19+
runs-on: [ "self-hosted", "overseas" ]
20+
steps:
21+
- name: Ensure workspace ownership
22+
run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE
23+
24+
# https://github.com/actions/checkout/blob/v3/README.md
25+
- name: Check out code
26+
uses: actions/checkout@v4
27+
with:
28+
token: ${{ secrets.MY_GITHUB_TOKEN }} # Use the secret as an environment variable
29+
fetch-depth: 0
30+
fetch-tags: true
31+
32+
- name: Prepare release body
33+
run: |
34+
if [[ $GITHUB_EVENT_NAME == 'create' ]]; then
35+
RELEASE_TAG=${GITHUB_REF#refs/tags/}
36+
if [[ $RELEASE_TAG == 'nightly' ]]; then
37+
PRERELEASE=true
38+
else
39+
PRERELEASE=false
40+
fi
41+
echo "Workflow triggered by create tag: $RELEASE_TAG"
42+
else
43+
RELEASE_TAG=nightly
44+
PRERELEASE=true
45+
echo "Workflow triggered by schedule"
46+
fi
47+
echo "RELEASE_TAG=$RELEASE_TAG" >> $GITHUB_ENV
48+
echo "PRERELEASE=$PRERELEASE" >> $GITHUB_ENV
49+
RELEASE_DATETIME=$(date --rfc-3339=seconds)
50+
echo Release $RELEASE_TAG created from $GITHUB_SHA at $RELEASE_DATETIME > release_body.md
51+
52+
- name: Move the existing mutable tag
53+
# https://github.com/softprops/action-gh-release/issues/171
54+
run: |
55+
git fetch --tags
56+
if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
57+
# Determine if a given tag exists and matches a specific Git commit.
58+
# actions/checkout@v4 fetch-tags doesn't work when triggered by schedule
59+
if [ "$(git rev-parse -q --verify "refs/tags/$RELEASE_TAG")" = "$GITHUB_SHA" ]; then
60+
echo "mutable tag $RELEASE_TAG exists and matches $GITHUB_SHA"
61+
else
62+
git tag -f $RELEASE_TAG $GITHUB_SHA
63+
git push -f origin $RELEASE_TAG:refs/tags/$RELEASE_TAG
64+
echo "created/moved mutable tag $RELEASE_TAG to $GITHUB_SHA"
65+
fi
66+
fi
67+
68+
- name: Create or overwrite a release
69+
# https://github.com/actions/upload-release-asset has been replaced by https://github.com/softprops/action-gh-release
70+
uses: softprops/action-gh-release@v2
71+
with:
72+
token: ${{ secrets.MY_GITHUB_TOKEN }} # Use the secret as an environment variable
73+
prerelease: ${{ env.PRERELEASE }}
74+
tag_name: ${{ env.RELEASE_TAG }}
75+
# The body field does not support environment variable substitution directly.
76+
body_path: release_body.md
77+
78+
- name: Set up QEMU
79+
uses: docker/setup-qemu-action@v3
80+
81+
- name: Set up Docker Buildx
82+
uses: docker/setup-buildx-action@v3
83+
84+
# https://github.com/marketplace/actions/docker-login
85+
- name: Login to Docker Hub
86+
uses: docker/login-action@v3
87+
with:
88+
username: infiniflow
89+
password: ${{ secrets.DOCKERHUB_TOKEN }}
90+
91+
# https://github.com/marketplace/actions/build-and-push-docker-images
92+
- name: Build and push full image
93+
uses: docker/build-push-action@v6
94+
with:
95+
context: .
96+
push: true
97+
tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}
98+
file: Dockerfile
99+
platforms: linux/amd64
100+
101+
# https://github.com/marketplace/actions/build-and-push-docker-images
102+
- name: Build and push slim image
103+
uses: docker/build-push-action@v6
104+
with:
105+
context: .
106+
push: true
107+
tags: infiniflow/ragflow:${{ env.RELEASE_TAG }}-slim
108+
file: Dockerfile
109+
build-args: LIGHTEN=1
110+
platforms: linux/amd64
111+
112+
- name: Build ragflow-sdk
113+
if: startsWith(github.ref, 'refs/tags/v')
114+
run: |
115+
cd sdk/python && \
116+
uv build
117+
118+
- name: Publish package distributions to PyPI
119+
if: startsWith(github.ref, 'refs/tags/v')
120+
uses: pypa/gh-action-pypi-publish@release/v1
121+
with:
122+
packages-dir: sdk/python/dist/
123+
password: ${{ secrets.PYPI_API_TOKEN }}
124+
verbose: true

‎.github/workflows/tests.yml

+23-16
Original file line numberDiff line numberDiff line change
@@ -49,29 +49,36 @@ jobs:
4949
fetch-depth: 0
5050
fetch-tags: true
5151

52-
- name: Build ragflow:dev-slim
52+
# https://github.com/astral-sh/ruff-action
53+
- name: Static check with Ruff
54+
uses: astral-sh/ruff-action@v2
55+
with:
56+
version: ">=0.8.2"
57+
args: "check --ignore E402"
58+
59+
- name: Build ragflow:nightly-slim
5360
run: |
5461
RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
55-
cp -r ${RUNNER_WORKSPACE_PREFIX}/huggingface.co ${RUNNER_WORKSPACE_PREFIX}/nltk_data ${RUNNER_WORKSPACE_PREFIX}/libssl*.deb ${RUNNER_WORKSPACE_PREFIX}/tika-server*.jar* ${RUNNER_WORKSPACE_PREFIX}/chrome* ${RUNNER_WORKSPACE_PREFIX}/cl100k_base.tiktoken .
5662
sudo docker pull ubuntu:22.04
57-
sudo docker build --progress=plain -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
63+
sudo docker build --progress=plain --build-arg LIGHTEN=1 --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly-slim .
5864
59-
- name: Build ragflow:dev
65+
- name: Build ragflow:nightly
6066
run: |
61-
sudo docker build --progress=plain -f Dockerfile -t infiniflow/ragflow:dev .
67+
sudo docker build --progress=plain --build-arg NEED_MIRROR=1 -f Dockerfile -t infiniflow/ragflow:nightly .
6268
63-
- name: Start ragflow:dev-slim
69+
- name: Start ragflow:nightly-slim
6470
run: |
71+
echo "RAGFLOW_IMAGE=infiniflow/ragflow:nightly-slim" >> docker/.env
6572
sudo docker compose -f docker/docker-compose.yml up -d
6673
67-
- name: Stop ragflow:dev-slim
74+
- name: Stop ragflow:nightly-slim
6875
if: always() # always run this step even if previous steps failed
6976
run: |
7077
sudo docker compose -f docker/docker-compose.yml down -v
7178
72-
- name: Start ragflow:dev
79+
- name: Start ragflow:nightly
7380
run: |
74-
echo "RAGFLOW_IMAGE=infiniflow/ragflow:dev" >> docker/.env
81+
echo "RAGFLOW_IMAGE=infiniflow/ragflow:nightly" >> docker/.env
7582
sudo docker compose -f docker/docker-compose.yml up -d
7683
7784
- name: Run sdk tests against Elasticsearch
@@ -82,7 +89,7 @@ jobs:
8289
echo "Waiting for service to be available..."
8390
sleep 5
8491
done
85-
cd sdk/python && poetry install && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
92+
cd sdk/python && uv sync --python 3.10 --frozen && uv pip install . && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
8693
8794
- name: Run frontend api tests against Elasticsearch
8895
run: |
@@ -92,15 +99,15 @@ jobs:
9299
echo "Waiting for service to be available..."
93100
sleep 5
94101
done
95-
cd sdk/python && poetry install && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
102+
cd sdk/python && uv sync --python 3.10 --frozen && uv pip install . && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
96103
97104
98-
- name: Stop ragflow:dev
105+
- name: Stop ragflow:nightly
99106
if: always() # always run this step even if previous steps failed
100107
run: |
101108
sudo docker compose -f docker/docker-compose.yml down -v
102109
103-
- name: Start ragflow:dev
110+
- name: Start ragflow:nightly
104111
run: |
105112
sudo DOC_ENGINE=infinity docker compose -f docker/docker-compose.yml up -d
106113
@@ -112,7 +119,7 @@ jobs:
112119
echo "Waiting for service to be available..."
113120
sleep 5
114121
done
115-
cd sdk/python && poetry install && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
122+
cd sdk/python && uv sync --python 3.10 --frozen && uv pip install . && source .venv/bin/activate && cd test/test_sdk_api && pytest -s --tb=short get_email.py t_dataset.py t_chat.py t_session.py t_document.py t_chunk.py
116123
117124
- name: Run frontend api tests against Infinity
118125
run: |
@@ -122,9 +129,9 @@ jobs:
122129
echo "Waiting for service to be available..."
123130
sleep 5
124131
done
125-
cd sdk/python && poetry install && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
132+
cd sdk/python && uv sync --python 3.10 --frozen && uv pip install . && source .venv/bin/activate && cd test/test_frontend_api && pytest -s --tb=short get_email.py test_dataset.py
126133
127-
- name: Stop ragflow:dev
134+
- name: Stop ragflow:nightly
128135
if: always() # always run this step even if previous steps failed
129136
run: |
130137
sudo DOC_ENGINE=infinity docker compose -f docker/docker-compose.yml down -v

‎.gitignore

+3-1
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,6 @@ rag/res/deepdoc
3535
sdk/python/ragflow.egg-info/
3636
sdk/python/build/
3737
sdk/python/dist/
38-
sdk/python/ragflow_sdk.egg-info/
38+
sdk/python/ragflow_sdk.egg-info/
39+
huggingface.co/
40+
nltk_data/

‎CONTRIBUTING.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Contribution guidelines
22

3-
This document offers guidlines and major considerations for submitting your contributions to RAGFlow.
3+
This document offers guidelines and major considerations for submitting your contributions to RAGFlow.
44

55
- To report a bug, file a [GitHub issue](https://github.com/infiniflow/ragflow/issues/new/choose) with us.
66
- For further questions, you can explore existing discussions or initiate a new one in [Discussions](https://github.com/orgs/infiniflow/discussions).

0 commit comments

Comments
 (0)