Sync changes from internal repo (#32)

* Copy repo

* Readd .env.example

Author: haoshan98

.env.example

@@ -6,6 +6,8 @@ TOGETHER_API_KEY=
 HYPERBOLIC_API_KEY=
 CEREBRAS_API_KEY=
 SAMBANOVA_API_KEY=
+DEEPSEEK_API_KEY=
+CUSTOM_API_KEY=
 
 # Service URLs
 DOCIO_URL=http://docio:6979/api/docio

.gitattributes

@@ -19,7 +19,7 @@
 # https://git-scm.com/docs/gitattributes#_text
 *.css text
 *.html text
-*.js text
+*.js* text
 *.md text
 *.py text
 *.sh text

.github/workflows/ci-win.yml

@@ -17,9 +17,23 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  check_changes:
+    name: Check for changes
+    runs-on: ubuntu-latest
+    outputs:
+      has-changes: ${{ steps.check.outputs.has-changes }}
+    steps:
+      - name: Check
+        id: check
+        uses: jiahuei/check-changes-action@v0
+        with:
+          watch-dirs: "clients/python/ services/api/ services/docio/ docker/ .github/"
+
   pyinstaller_electron_app:
     name: PyInstaller JamAIBase Electron App Compilation
     runs-on: windows-11-desktop
+    needs: check_changes
+    if: needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push'
     timeout-minutes: 60
 
     steps:
@@ -98,6 +112,8 @@ jobs:
   pyinstaller_api:
     name: PyInstaller API Service Compilation
     runs-on: windows-11-desktop
+    needs: check_changes
+    if: needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push'
     timeout-minutes: 60
 
     steps:
@@ -179,6 +195,8 @@ jobs:
   pyinstaller_docio:
     name: PyInstaller DocIO Service Compilation
     runs-on: windows-11-desktop
+    needs: check_changes
+    if: needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push'
     timeout-minutes: 60
 
     steps:

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: CI
+name: CI (OSS)
 
 on:
   pull_request:
@@ -18,10 +18,38 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  check_changes:
+    name: Check for changes
+    runs-on: ubuntu-latest
+    outputs:
+      has-changes: ${{ steps.check.outputs.has-changes }}
+    steps:
+      - name: Check
+        id: check
+        uses: jiahuei/check-changes-action@v0
+        with:
+          watch-dirs: "clients/python/ services/api/ docker/ .github/"
+
+  sdk_tests_noop:
+    # This job is needed so that status checks can still pass
+    # This is because strategy matrix is evaluated after if condition
+    name: SDK unit tests
+    runs-on: ubuntu-latest
+    needs: check_changes
+    if: ${{ !(needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push') }}
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    timeout-minutes: 2
+    steps:
+      - name: No-op
+        run: echo Tests skipped !!!
+
   sdk_tests:
     name: SDK unit tests
     runs-on: ubuntu-latest-l
-    # runs-on: namespace-profile-ubuntu-latest-8cpu-16gb-96gb
+    needs: check_changes
+    if: needs.check_changes.outputs.has-changes == 'true' || github.event_name == 'push'
     strategy:
       matrix:
         python-version: ["3.10"]
@@ -42,18 +70,23 @@ jobs:
         run: |
           git --version
 
+      - name: Check Docker Version
+        run: docker version
+
+      - name: Check Docker Compose Version
+        run: docker compose version
+
       - name: Remove cloud-only modules and install Python client
         run: |
           set -e
           bash scripts/remove_cloud_modules.sh
           cd clients/python
           python -m pip install .[test]
 
-      - name: Check Docker Version
-        run: docker version
-
-      - name: Check Docker Compose Version
-        run: docker compose version
+      - name: Install ffmpeg
+        run: |
+          set -e
+          sudo apt-get update -qq && sudo apt-get install ffmpeg libavcodec-extra -y
 
       - name: Authenticating to the Container registry
         run: echo $JH_PAT | docker login ghcr.io -u [email protected] --password-stdin
@@ -87,13 +120,14 @@ jobs:
           TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
           COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
           HYPERBOLIC_API_KEY: ${{ secrets.HYPERBOLIC_API_KEY }}
+          CUSTOM_API_KEY: ${{ secrets.CUSTOM_API_KEY }}
 
       - name: Launch services (OSS)
         id: launch_oss
         timeout-minutes: 20
         run: |
           set -e
-          docker compose -p jamai -f docker/compose.cpu.yml --profile minio up --quiet-pull -d --wait
+          docker compose -p jamai -f docker/compose.cpu.yml --profile minio --profile kopi up --quiet-pull -d --wait
         env:
           COMPOSE_DOCKER_CLI_BUILD: 1
           DOCKER_BUILDKIT: 1
@@ -116,7 +150,7 @@ jobs:
             --junitxml=junit/test-results-${{ matrix.python-version }}.xml \
             --cov-report=xml \
             --no-flaky-report \
-            clients/python/tests/oss
+            clients/python/tests/oss/
 
       - name: Inspect owl logs if Python SDK tests failed
         if: failure() &&  steps.python_sdk_test_oss.outcome == 'failure'
@@ -170,11 +204,6 @@ jobs:
             --no-flaky-report \
             clients/python/tests/oss/test_file.py
 
-      - name: Inspect owl logs if Python SDK tests failed
-        if: failure() &&  steps.python_sdk_test_oss_file.outcome == 'failure'
-        timeout-minutes: 1
-        run: docker exec jamai-owl-1 cat /app/api/logs/owl.log
-
   lance_tests:
     name: Lance tests
     runs-on: ubuntu-latest

.github/workflows/github_bot.yml

@@ -0,0 +1,44 @@
+name: JambuBot
+
+on:
+  issues:
+    types: [opened, edited]
+  pull_request:
+    types: [opened, synchronize]
+
+# Cancel in-progress CI jobs if there is a new push
+# https://stackoverflow.com/a/72408109
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  github-bot:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          submodules: true # Ensure submodules are checked out
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.18"
+
+      - name: Launching JambuBot
+        run: |
+          cd examples/github-bot/github-bot/src
+          go build -o github_bot cmd/main.go
+          ./github_bot
+        env:
+          TRIAGE_BOT_APP_ID: ${{ secrets.TRIAGE_BOT_APP_ID }}
+          TRIAGE_BOT_INSTALLATION_ID: ${{ secrets.TRIAGE_BOT_INSTALLATION_ID }}
+          TRIAGE_BOT_PRIVATE_KEY: ${{ secrets.TRIAGE_BOT_PRIVATE_KEY }}
+          TRIAGE_BOT_JAMAI_KEY: ${{ secrets.TRIAGE_BOT_JAMAI_KEY }}
+          TRIAGE_BOT_JAMAI_PROJECT_ID: ${{ secrets.TRIAGE_BOT_JAMAI_PROJECT_ID }}
+          TRIAGE_BOT_NAME: ${{github.actor}}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          GITHUB_EVENT_PATH: ${{ github.event_path }}

.prettierignore

@@ -1,39 +1,41 @@
+# OS
+thumbs.db
+.DS_Store
+
+# Internal references, dependencies, temporary folders & files
+.env
+archive/
+**/__ref__/
+*.log
+*.lock
+*.db
+*.parquet
+
 # Python
 __pycache__/
-*.py[cod]
+*.py*
 *$py.class
 *.egg-info
 .pytest_cache
 .ipynb_checkpoints
 venv/
-
-# Frontend
-services/app
-clients/typescript
-
-# Test files
 clients/python/tests/**/*
 
-# Internal references, dependencies, temporary folders & files
-archive/
-/dependencies/
-logs/
-*.log
+# pip
+**/build/
 
 # pytest-cov
 **/.coverage*
 /junit
 /htmlcov
 /coverage.xml
 
-# pip
-**/build/
+# jest-cov
+**/coverage/*
 
-# Docs
-/docs/source/generated/
-/docs/source/api/generated/
-/docs/source/specs/
+# JavaScript
+**/node_modules/
 
-# OS
-thumbs.db
-.DS_Store
+# Frontend
+services/app
+clients/typescript

CHANGELOG.md

@@ -16,32 +16,79 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
-Backend - owl (API server)
+### ADDED
+
+Python SDK - jamaibase
 
-- Fix bge-small embedding size (1024 -> 384)
-- Correctly filter models at auth level
-- Fix ollama model deployment config
+- Add `CodeGenConfig` for python code execution #446
 
-Frontend
+TS SDK - jamaibase
+
+UI
 
-- Added support for multiple multiturn columns in Chat table chat view.
-- Added multiturn chat toggle to column settings.
+- Support chat mode multiturn option in add column and column resize #451
+- Support `audio` data type #457
 
-Docker
+Backend - owl (API server)
 
-- Added Mac Apple Silicon `compose.mac.yml`
-- Update `ollama.yml` to use Qwen2.5 3B
-- Fix ollama default config
+- GenTable
+  - Support `audio` input column and data type #443
+  - Support python code execution column #446
+  - **Breaking**: Add `Page` column to knowledge table #464
+- LLM
+  - Support function calling # 435
+  - Support DeepSeek models #466
+- Billing
+  - Include background tasks for processing billing events #462
+- Auth
+  - Support specific user role in organization invite #446
+  - Include background tasks for setting project updated at datetime #462
+- Handle and allow setting of file upload size limits for `embed file`, `image` and `audio` file types #443
 
-## [v0.3.1] (2024-11-26)
+CI/CD
 
-This is a bug fix release for frontend code. SDKs are not affected.
+- Added a new CI workflow for cloud environments in`.github/workflows/ci.cloud.yml` #440
+- Add dummy test job to pass status checks if skipped #468
+- Added a `check_changes` job to the CI workflows to conditionally run SDK tests based on changes. #462
 
 ### CHANGES / FIXED
 
-Frontend
+Python SDK - jamaibase
+
+TS SDK - jamaibase
 
-- Enable Projects for OSS
+- Update the `uploadFile` method in `index.ts` to remove the trailing slash from the API endpoint #462
+
+UI
+
+- Remove unnecessary load function rerunning on client navigation #454
+- Add more export options with confirmation #459
+- Obfuscated external keys and credit values for non-admin users in the `+layout.server.ts` to enhance security and privacy #459
+- Update `FileSelect.svelte` and `NewRow.svelte` to remove trailing slashes from the file upload API endpoint #462
+- Bug fixes:
+  - Fix chat table scrollbar not showing issue #459
+  - Fix keyboard navigation #459
+  - Fix inappropriate model not showing issue in knowledge table column settings #459
+
+Backend - owl (API server)
+
+- GenTable
+  - **Breaking**: Change `file` data type to `image` data type #460
+- LLM
+  - Handle usage tracking and improve error handling #462
+  - Bug fixes
+    - Fix model config embedding size #441
+    - Fix bug with default model choosing invalid models #442
+    - Fix regen sequences issue after columns reordering #455
+
+CI/CD
+
+- Dockerfile: Added `ffmpeg` installation for audio processing. #443
+- Dependency Updates:
+  - Set `litellm` to version `1.50.0` #443
+  - Add `pydub` as a dependency for audio processing #443
+
+### REMOVED
 
 ## [v0.3] (2024-11-20)
 
@@ -427,7 +474,7 @@ Backend - owl (API server)
   - Windows: StreamResponse from FastAPI accumulates all SSE before yielding everything all at once to the client Fixes #145
   - Enabled scanned pdf upload. Fixes #131
 - Dependencies
-  - Support forked version of `unstructured-client==0.24.1`, changed nest-asyncio to ThreadPool, fixed the conflict with uvloop
+  - Support forked version of `unstructured-client==0.24.1`, changed `nest-asyncio` to `ThreadPool`, fixed the conflict with `uvloop`
   - Added `tenacity`, `pandas`
   - Bumped dependency versions
 
@@ -441,7 +488,7 @@ Backend - Admin (cloud)
 
 - Improve insufficient credit error message: include quota/usage type in the message
 - Storage usage update is now a background process; fixes #87
-- Allow dot in the middle for project name and organization name.
+- Allow dot in the middle for project name and organization name.
 - Update `models.json` in `set_model_config()`
 - Billing: Don't include Lance version directories in storage usage computation
 - Bug fixes