From b59203eaec84c98fa17b251362eb5cf0b3f67933 Mon Sep 17 00:00:00 2001 From: Bohan Cheng <47214785+cbh778899@users.noreply.github.com> Date: Tue, 20 Aug 2024 16:03:55 +1000 Subject: [PATCH] Add CLI (#60) * add CLI Signed-off-by: cbh778899 * remove setup executable Signed-off-by: cbh778899 * fix environment variable not set bug Signed-off-by: cbh778899 * add setup executable Signed-off-by: cbh778899 * add readme for CLI Signed-off-by: cbh778899 --------- Signed-off-by: cbh778899 --- .dockerignore | 4 +- .env | 8 +- .gitignore | 3 +- Makefile | 26 +- README.md | 12 + database/index.js | 2 +- docker-compose-adv.yaml | 46 +++ setup/Dockerfile | 7 + setup/config.h | 46 +++ setup/default_config.h | 46 +++ setup/setup.c | 694 ++++++++++++++++++++++++++++++++++++++++ setup/setup.sh | 8 + setup/setup_types.h | 609 +++++++++++++++++++++++++++++++++++ 13 files changed, 1496 insertions(+), 15 deletions(-) create mode 100644 docker-compose-adv.yaml create mode 100644 setup/Dockerfile create mode 100644 setup/config.h create mode 100644 setup/default_config.h create mode 100644 setup/setup.c create mode 100644 setup/setup.sh create mode 100644 setup/setup_types.h diff --git a/.dockerignore b/.dockerignore index 23e3147..4588173 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,4 +12,6 @@ eslint.config.mjs LICENSE volumes docker-compose* -Makefile \ No newline at end of file +Makefile +setup +generate_production_env.html \ No newline at end of file diff --git a/.env b/.env index 1929ba8..266e34b 100644 --- a/.env +++ b/.env @@ -1,15 +1,15 @@ APP_PORT=8000 +APP_EXPOSE_PORT=8000 ENG_ACCESS_PORT=8080 MODEL_SAVE_PATH=volumes/models INFERENCE_ENG=llamacpp INFERENCE_ENG_PORT=8080 -INFERENCE_ENG_VERSION=server--b1-2321a5e +INFERENCE_ENG_VERSION=server--b1-27d4b7c NUM_CPU_CORES=8.00 -NUM_THREADS_COUNT=8 +NUM_THREADS_COUNT=8.00 EMBEDDING_ENG=embedding_eng EMBEDDING_ENG_PORT=8081 NUM_CPU_CORES_EMBEDDING=4.00 +NUM_THREAD_COUNTS_EMBEDDING=4.00 LANGUAGE_MODEL_NAME=Phi3-mini-4k-instruct-Q4.gguf -LANGUAGE_MODEL_URL=https://huggingface.co/aisuko/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi3-mini-4k-instruct-Q4.gguf?download=true EMBEDDING_MODEL_NAME=all-MiniLM-L6-v2-Q4_K_M-v2.gguf -EMBEDDING_MODEL_URL=https://huggingface.co/aisuko/all-MiniLM-L6-v2-gguf/resolve/main/all-MiniLM-L6-v2-Q4_K_M-v2.gguf?download=true diff --git a/.gitignore b/.gitignore index af209fa..a7f452c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules .git volumes -__pycache__ \ No newline at end of file +__pycache__ +setup/setup \ No newline at end of file diff --git a/Makefile b/Makefile index 97d1e2a..223f665 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ # project related PROJECT_NAME:=voyager -CONTAINER_NAME:=voyager:v0.1.0 +CONTAINER_NAME:=voyager:v0.2.0 APP_PORT:=8000 +APP_EXPOSE_PORT:=8000 # compose build related ENV_FILE:=.env @@ -10,15 +11,16 @@ MODEL_SAVE_PATH:=volumes/models INFERENCE_ENG:=llamacpp INFERENCE_ENG_PORT:=8080 -INFERENCE_ENG_VERSION:=server--b1-2321a5e +INFERENCE_ENG_VERSION:=server--b1-27d4b7c NUM_CPU_CORES:=8.00 -NUM_THREADS_COUNT:=8 +NUM_THREADS_COUNT:=8.00 EMBEDDING_ENG:=embedding_eng EMBEDDING_ENG_PORT:=8081 NUM_CPU_CORES_EMBEDDING:=4.00 -LANGUAGE_MODEL_NAME:=Phi3-mini-4k-instruct-Q4.gguf -LANGUAGE_MODEL_URL:=https://huggingface.co/aisuko/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi3-mini-4k-instruct-Q4.gguf?download=true +NUM_THREAD_COUNTS_EMBEDDING:=4.00 +LANGUAGE_MODEL_NAME:=ft-smollm-135M-instruct-on-hf-ultrafeedback-f16.gguf +LANGUAGE_MODEL_URL:=https://huggingface.co/aisuko/ft-smollm-135M-instruct-on-hf-ultrafeedback-gguf/resolve/main/ft-smollm-135M-instruct-on-hf-ultrafeedback-f16.gguf EMBEDDING_MODEL_NAME:=all-MiniLM-L6-v2-Q4_K_M-v2.gguf EMBEDDING_MODEL_URL:=https://huggingface.co/aisuko/all-MiniLM-L6-v2-gguf/resolve/main/all-MiniLM-L6-v2-Q4_K_M-v2.gguf?download=true @@ -35,6 +37,7 @@ run: build .PHONY: env env: @echo "APP_PORT=$(APP_PORT)"> $(ENV_FILE) + @echo "APP_EXPOSE_PORT=$(APP_EXPOSE_PORT)">> $(ENV_FILE) @echo "ENG_ACCESS_PORT=$(ENG_ACCESS_PORT)">> $(ENV_FILE) @echo "MODEL_SAVE_PATH=$(MODEL_SAVE_PATH)">> $(ENV_FILE) @echo "INFERENCE_ENG=$(INFERENCE_ENG)">> $(ENV_FILE) @@ -45,10 +48,9 @@ env: @echo "EMBEDDING_ENG=$(EMBEDDING_ENG)">> $(ENV_FILE) @echo "EMBEDDING_ENG_PORT=$(EMBEDDING_ENG_PORT)">> $(ENV_FILE) @echo "NUM_CPU_CORES_EMBEDDING=$(NUM_CPU_CORES_EMBEDDING)">> $(ENV_FILE) + @echo "NUM_THREAD_COUNTS_EMBEDDING=$(NUM_THREAD_COUNTS_EMBEDDING)">> $(ENV_FILE) @echo "LANGUAGE_MODEL_NAME=$(LANGUAGE_MODEL_NAME)">> $(ENV_FILE) - @echo "LANGUAGE_MODEL_URL=$(LANGUAGE_MODEL_URL)">> $(ENV_FILE) @echo "EMBEDDING_MODEL_NAME=$(EMBEDDING_MODEL_NAME)">> $(ENV_FILE) - @echo "EMBEDDING_MODEL_URL=$(EMBEDDING_MODEL_URL)">> $(ENV_FILE) .PHONY: model-prepare model-prepare: @@ -83,4 +85,12 @@ stop: .PHONY: pytest pytest: - @python3 -m pytest -v \ No newline at end of file + @python3 -m pytest -v + +######################################################################################### +# setup + +.PHONY: setup +setup: + gcc setup/setup.c -o setup/setup + setup/setup \ No newline at end of file diff --git a/README.md b/README.md index bfcda37..d90e406 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,18 @@ The project is OpenAI-like API service of SkywardAI ecosystem. https://github.com/user-attachments/assets/2b8f1ea7-0aca-44ea-b218-eff8e1769729 +### CLI 💥 +Introducing our new CLI tool! +> Make sure you can normally run `make`, `docker compose`, `gcc`, `sh` in your host machine. + +Simply run `make setup` in the root folder to compile & run the CLI tool. + +Don't want to set? Directly go to `Save & Build` menu in it and use `Build and start the server` option to load the app in default settings. + +No gcc compiler? You can choose to compile the file `/setup/setup.c` yourself. + +Explore it yourself to find more settings! + ### Local Machine * Please make sure you installed `Node.js` on your local machine. diff --git a/database/index.js b/database/index.js index d171852..7e0d1cd 100644 --- a/database/index.js +++ b/database/index.js @@ -42,7 +42,7 @@ export async function initDB(force = false) { // create or re-open api key table await db.createEmptyTable(API_KEY_TABLE, new Schema([ new Field("api_key", new Utf8()), - new Field("usage", new Int32()), + new Field("usage", new Int32()) ]), open_options); } diff --git a/docker-compose-adv.yaml b/docker-compose-adv.yaml new file mode 100644 index 0000000..a48b009 --- /dev/null +++ b/docker-compose-adv.yaml @@ -0,0 +1,46 @@ +services: + llamacpp: + container_name: ${INFERENCE_ENG} + image: gclub/llama.cpp:${INFERENCE_ENG_VERSION} + restart: always + deploy: # https://github.com/compose-spec/compose-spec/blob/master/deploy.md + resources: + reservations: + cpus: "${NUM_CPU_CORES}" + volumes: + - "${DOCKER_VOLUME_DIRECTORY:-.}/${MODEL_SAVE_PATH}:/models" + expose: + - ${ENG_ACCESS_PORT} + ports: + - ${INFERENCE_ENG_PORT}:${ENG_ACCESS_PORT} + command: ["-m", "models/${LANGUAGE_MODEL_NAME}","-t","${NUM_THREADS_COUNT}","-c","8192"] + + embedding_eng: + container_name: ${EMBEDDING_ENG} + image: gclub/llama.cpp:${INFERENCE_ENG_VERSION} + restart: always + deploy: # https://github.com/compose-spec/compose-spec/blob/master/deploy.md + resources: + reservations: + cpus: "${NUM_CPU_CORES_EMBEDDING}" + volumes: + - "${DOCKER_VOLUME_DIRECTORY:-.}/${MODEL_SAVE_PATH}:/models" + expose: + - ${ENG_ACCESS_PORT} + ports: + - ${EMBEDDING_ENG_PORT}:${ENG_ACCESS_PORT} + command: ["-m", "models/${EMBEDDING_MODEL_NAME}","--embeddings","--pooling","mean","-t","${NUM_THREAD_COUNTS_EMBEDDING}","-c","512"] + + voyager: + container_name: voyager + restart: always + build: + dockerfile: setup/Dockerfile + context: . + expose: + - ${APP_EXPOSE_PORT} + ports: + - ${APP_EXPOSE_PORT}:${APP_PORT} + depends_on: + - llamacpp + - embedding_eng diff --git a/setup/Dockerfile b/setup/Dockerfile new file mode 100644 index 0000000..6df3c87 --- /dev/null +++ b/setup/Dockerfile @@ -0,0 +1,7 @@ +FROM node:20.15.1-slim +WORKDIR /app +COPY . . + +HEALTHCHECK --interval=300s --timeout=30s --start-period=5s --retries=3 CMD [ "node", "healthy-check.js" ] +RUN npm install -g pnpm && pnpm install +ENTRYPOINT [ "npm", "start" ] \ No newline at end of file diff --git a/setup/config.h b/setup/config.h new file mode 100644 index 0000000..b5b3bed --- /dev/null +++ b/setup/config.h @@ -0,0 +1,46 @@ +#include "setup_types.h" + +#ifndef SETUP_CONFIG_H +#define SETUP_CONFIG_H + +#define INFERENCE_MODEL_NAME "Phi3-mini-4k-instruct-Q4.gguf" +#define INFERENCE_MODEL_URL "https://huggingface.co/aisuko/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi3-mini-4k-instruct-Q4.gguf" +#define INFERENCE_CPU_CORES 8.00 +#define INFERENCE_THREAD_COUNTS 8.00 + +#define EMBEDDING_CPU_CORES 4.00 +#define EMBEDDING_THREAD_COUNTS 4.00 + +#define ALLOW_ORIGIN_NAME "*" + +#define HTTPS_ENABLED 0 +#define HTTPS_CERT_PATH_HOST "*" +#define HTTPS_CERT_PATH_CONTAINER "*" +#define HTTPS_CERT_NAME "cert.pem" +#define HTTPS_PRIVKEY_NAME "privkey.pem" +#define HTTPS_CA_NAME "chain.pem" +#define APP_EXPOSE_PORT "8000" + +#define PLUGIN_ENABLED 0 + +#define SYSTEM_INSTRUCTION "*" + +#define STATIC_API_KEY_ENABLED 0 +#define STATIC_API_KEY "*" + +#define DEFAULT_DATASET_ENABLED 0 +#define DEFAULT_DATASET_NAME "production_dataset" + +#define API_INDEX_DOC_ENABLED 1 +#define API_INDEX_STATS_ENABLED 1 +#define API_INDEX_HEALTHY_ENABLED 1 +#define API_INFERENCE_COMP_ENABLED 1 +#define API_INFERENCE_RAG_ENABLED 1 +#define API_TOKEN_ENABLED 1 +#define API_EMBEDDING_CALC_ENABLED 1 +#define API_EMBEDDING_DS_ENABLED 1 +#define API_VERSION_ENABLED 1 + +#define DEV_MODE_ENABLED 0 + +#endif \ No newline at end of file diff --git a/setup/default_config.h b/setup/default_config.h new file mode 100644 index 0000000..1d01dd8 --- /dev/null +++ b/setup/default_config.h @@ -0,0 +1,46 @@ +#include "setup_types.h" + +#ifndef SETUP_DEFAULT_CONFIG_H +#define SETUP_DEFAULT_CONFIG_H + +#define DEFAULT_INFERENCE_MODEL_NAME MODEL_PHI_NAME +#define DEFAULT_INFERENCE_MODEL_URL MODEL_PHI_URL +#define DEFAULT_INFERENCE_CPU_CORES 8.00 +#define DEFAULT_INFERENCE_THREAD_COUNTS 8.00 + +#define DEFAULT_EMBEDDING_CPU_CORES 4.00 +#define DEFAULT_EMBEDDING_THREAD_COUNTS 4.00 + +#define DEFAULT_ALLOW_ORIGIN_NAME "*" + +#define DEFAULT_HTTPS_ENABLED 0 +#define DEFAULT_HTTPS_CERT_PATH_HOST "*" +#define DEFAULT_HTTPS_CERT_PATH_CONTAINER "*" +#define DEFAULT_HTTPS_CERT_NAME "cert.pem" +#define DEFAULT_HTTPS_PRIVKEY_NAME "privkey.pem" +#define DEFAULT_HTTPS_CA_NAME "chain.pem" +#define DEFAULT_APP_EXPOSE_PORT "8000" + +#define DEFAULT_PLUGIN_ENABLED 0 + +#define DEFAULT_SYSTEM_INSTRUCTION "*" + +#define DEFAULT_STATIC_API_KEY_ENABLED 0 +#define DEFAULT_STATIC_API_KEY "*" + +#define DEFAULT_DEFAULT_DATASET_ENABLED 0 +#define DEFAULT_DEFAULT_DATASET_NAME "production_dataset" + +#define DEFAULT_API_INDEX_DOC_ENABLED 1 +#define DEFAULT_API_INDEX_STATS_ENABLED 1 +#define DEFAULT_API_INDEX_HEALTHY_ENABLED 1 +#define DEFAULT_API_INFERENCE_COMP_ENABLED 1 +#define DEFAULT_API_INFERENCE_RAG_ENABLED 1 +#define DEFAULT_API_TOKEN_ENABLED 1 +#define DEFAULT_API_EMBEDDING_CALC_ENABLED 1 +#define DEFAULT_API_EMBEDDING_DS_ENABLED 1 +#define DEFAULT_API_VERSION_ENABLED 1 + +#define DEFAULT_DEV_MODE_ENABLED 0 + +#endif \ No newline at end of file diff --git a/setup/setup.c b/setup/setup.c new file mode 100644 index 0000000..822ce4f --- /dev/null +++ b/setup/setup.c @@ -0,0 +1,694 @@ +#include "setup_types.h" + +// ===============================VARS=============================== +char* inference_model_name = INFERENCE_MODEL_NAME; +char* inference_model_url = INFERENCE_MODEL_URL; +float inference_cpu_cores = INFERENCE_CPU_CORES; +float inference_thread_counts = INFERENCE_THREAD_COUNTS; +float embedding_cpu_cores = EMBEDDING_CPU_CORES; +float embedding_thread_counts = EMBEDDING_THREAD_COUNTS; + +char* allow_origin_name = ALLOW_ORIGIN_NAME; + +int https_enabled = HTTPS_ENABLED; +char* https_cert_path_host = HTTPS_CERT_PATH_HOST; +char* https_cert_path_container = HTTPS_CERT_PATH_CONTAINER; +char* https_cert_name = HTTPS_CERT_NAME; +char* https_privkey_name = HTTPS_PRIVKEY_NAME; +char* https_ca_name = HTTPS_CA_NAME; +char* app_expose_port = APP_EXPOSE_PORT; + +int plugin_enabled = PLUGIN_ENABLED; + +char* system_instruction = SYSTEM_INSTRUCTION; + +int static_api_key_enabled = STATIC_API_KEY_ENABLED; +char* static_api_key_value = STATIC_API_KEY; + +int default_dataset_enabled = DEFAULT_DATASET_ENABLED; +char* default_dataset_name = DEFAULT_DATASET_NAME; + +int api_index_doc_enabled = API_INDEX_DOC_ENABLED; +int api_index_stats_enabled = API_INDEX_STATS_ENABLED; +int api_index_healthy_enabled = API_INDEX_HEALTHY_ENABLED; +int api_inference_comp_enabled = API_INFERENCE_COMP_ENABLED; +int api_inference_rag_enabled = API_INFERENCE_RAG_ENABLED; +int api_token_enabled = API_TOKEN_ENABLED; +int api_embedding_calc_enabled = API_EMBEDDING_CALC_ENABLED; +int api_embedding_ds_enabled = API_EMBEDDING_DS_ENABLED; +int api_version_enabled = API_VERSION_ENABLED; + +int dev_mode_enabled = DEV_MODE_ENABLED; + +// ===============================FUNCTION DECLARATION=============================== + +void logMenu(char*); +void logMessage(char*); +void handleMenuSelection(); +char getSelectedOption(); + +void setNumberFields(char*, char*, char*, char*, int, int, float*); +void setStringFields(char*, char*, char*, char**); +void setStringFieldsWithLength(char*, char*, char*, char**, int); +void setYesNoFields(char*, int*); + + +void engineCPUSettings(char*, float*); +void engineThreadSettings(char*, float*); + +void inferenceEngineSettings(); +void inferenceEngineModelSettings(); +void embeddingEngineSettings(); + +void httpsSettings(); +void httpsCertPathSettings(); +void httpsCertNameSettings(); + +void staticAPIKeySettings(); + +void defaultDatasetSettings(); + +void logAPIMenu(); +void apiSettings(); + +void buildSettings(); +void saveSettings(int); +void saveDefaultSettings(); + +// ===============================MAIN=============================== +int main() { + while (1) { + logMenu(MENU_TEXT); + handleMenuSelection(); + } + return 0; +} + +// ===============================MAIN MENU=============================== +void logMenu(char* menu) { + fflush(stdout); + system("clear"); + printf("%s", menu); +} + +void logMessage(char* message) { + printf(MESSAGE_MENU, message); + char s[10]; + fgets(s, 10, stdin); +} + +void handleMenuSelection() { + switch (getSelectedOption()) { + case 'a': case 'A': + inferenceEngineSettings(); break; + case 'b': case 'B': + embeddingEngineSettings(); break; + case 'c': case 'C': + setStringFields( + ALLOW_ORIGIN_MENU, + ALLOW_ORIGIN_SUCCESS_MSG, + "Allowed Origin Name:", + &allow_origin_name + ); break; + case 'd': case 'D': + httpsSettings(); + break; + case 'e': case 'E': + setYesNoFields(PLUGIN_MENU, &plugin_enabled); + break; + case 'f': case 'F': + setStringFieldsWithLength( + SYSTEM_INSTRUCTION_MENU, + "Set System Instruction", + "System instruction:", + &system_instruction, 500 + ); break; + case 'g': case 'G': + defaultDatasetSettings(); + break; + case 'h': case 'H': + staticAPIKeySettings(); + break; + case 'i': case 'I': + apiSettings(); + break; + case 'j': case 'J': + setYesNoFields(DEV_MODE_MENU, &dev_mode_enabled); break; + case 'k': case 'K': + buildSettings(); break; + case 'q': case 'Q': case ESC: + puts("Exit."); + exit(0); + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } +} + +char getSelectedOption() { + char selection[10]; + fgets(selection, 10, stdin); + return selection[0]; +} + +void setNumberFields( + char* log_message, char* selected_message, + char* success_message, char* failed_message, + int min, int max, + float* var_to_set +) { + while (1) { + logMenu(log_message); + char number[10]; + fgets(number, 10, stdin); + if(number[0] == 'q') return; + + float num = atof(number); + printf("\n%.2f %s\n\n", num, selected_message); + if(num <= min || (max >= 0 && num >= max)) { + logMessage(failed_message); + } else { + logMessage(success_message); + *var_to_set = num; + return; + } + } +} + +void setStringFieldsWithLength( + char* menu_text, char* success_message, + char* set_message, char** string_to_change, + int max_length +) { + logMenu(menu_text); + char name[max_length]; + fgets(name, max_length, stdin); + if(strcmp(name, "q\n") == 0) return; + + int len = strlen(name); + name[len - 1] = '\0'; + asprintf(string_to_change, "%s", (char*)name); + + printf("\n%s %s\n\n", set_message, *string_to_change); + logMessage(success_message); +} + +void setStringFields( + char* menu_text, char* success_message, + char* set_message, char** string_to_change +) { + setStringFieldsWithLength( + menu_text, success_message, + set_message, string_to_change, + 100 + ); +} + +void setYesNoFields(char* menu_text, int* int_to_set) { + while(1) { + logMenu(menu_text); + switch(getSelectedOption()) { + case 'y': case 'Y': + *int_to_set = 1; + return; + case 'n': case 'N': + *int_to_set = 0; + return; + case 'q': case 'Q': case ESC: + return; + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } + } +} + +// ===============================SUB MENUS=============================== + +// ===============================Llama.cpp Engines=============================== +void engineCPUSettings(char* menu_message, float* var_to_set) { + setNumberFields( + menu_message, + "Cores Selected.", + "Set CPU cores Success.", + "CPU cores is invalid!", + 0, -1, var_to_set + ); +} + +void engineThreadSettings(char* menu_message, float* var_to_set) { + setNumberFields( + menu_message, + "Threads Selected.", + "Set Thread num Success.", + "Thread num is invalid!", + 0, -1, var_to_set + ); +} + +// ===============================INFERENCE=============================== +void inferenceEngineSettings() { + while (1) { + logMenu(INFERENCE_ENG_MENU); + switch (getSelectedOption()) { + case 'a': case 'A': + inferenceEngineModelSettings(); + break; + case 'b': case 'B': + engineCPUSettings(INFERENCE_ENG_CPU, &inference_cpu_cores); + break; + case 'c': case 'C': + engineThreadSettings(INFERENCE_ENG_THREAD, &inference_thread_counts); + break; + case 'q': case 'Q': case ESC: + return; + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } + } + +} + +void inferenceEngineModelSettings() { + while (1) { + logMenu(INFERENCE_ENG_MODEL); + switch (getSelectedOption()) { + case MODEL_PHI: + inference_model_name = MODEL_PHI_NAME; + inference_model_url = MODEL_PHI_URL; + logMessage("Selected Model Phi3"); + return; + case MODEL_SMOLLM: + inference_model_name = MODEL_SMOLLM_NAME; + inference_model_url = MODEL_SMOLLM_URL; + logMessage("Selected Model Smollm"); + return; + case 'q': case ESC: + return; + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } + } +} + +// ===============================EMBEDDING=============================== +void embeddingEngineSettings() { + while (1) { + logMenu(EMBEDDING_ENG_MENU); + switch (getSelectedOption()) { + case 'a': case 'A': + engineCPUSettings(EMBEDDING_ENG_CPU, &embedding_cpu_cores); + break; + case 'b': case 'B': + engineThreadSettings(EMBEDDING_ENG_THREAD, &embedding_thread_counts); + break; + case 'q': case 'Q': case ESC: + return; + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } + } +} +// ===============================HTTPS=============================== +void httpsSettings() { + while(1) { + logMenu(HTTPS_MENU); + switch(getSelectedOption()) { + case 'a': case 'A': + setYesNoFields(HTTPS_AVAIL, &https_enabled); + break; + case 'b': case 'B': + httpsCertPathSettings(); + break; + case 'c': case 'C': + httpsCertNameSettings(); + break; + case 'd': case 'D': + setStringFields( + APP_OPEN_PORT_MENU, + "Set Expose Port Success", + "Expose port:", + &app_expose_port + ); + break; + case 'q': case 'Q': case ESC: + return; + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } + } +} + +void httpsCertPathSettings() { + while(1) { + logMenu(HTTPS_CERT_LOC); + switch(getSelectedOption()) { + case 'a': case 'A': + setStringFields( + HTTPS_CERT_LOC_HOST, + "Successfully Set Location", + "Path on host machine:", + &https_cert_path_host + ); + break; + case 'b': case 'B': + setStringFields( + HTTPS_CERT_LOC_CONTAINER, + "Successfully Set Location", + "Path in container:", + &https_cert_path_container + ); + break; + case 'c': case 'C': + setStringFields( + HTTPS_CERT_LOC_CONTAINER, + "Successfully Set Location", + "Path on both:", + &https_cert_path_container + ); + asprintf(&https_cert_path_host, "%s", https_cert_path_container); + break; + case 'q': case 'Q': case ESC: + return; + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } + } +} + +void httpsCertNameSettings() { + while(1) { + logMenu(HTTPS_CERT_NAME_MENU); + switch(getSelectedOption()) { + case 'a': case 'A': + setStringFields( + HTTPS_CERT_NAME_CERT, + "Set CERT Name Success", + "SSL/TSL certificate name:", + &https_cert_name + ); + break; + case 'b': case 'B': + setStringFields( + HTTPS_CERT_NAME_KEY, + "Set PRIVKEY Name Success", + "SSL/TSL private key name:", + &https_privkey_name + ); + break; + case 'c': case 'C': + setStringFields( + HTTPS_CERT_NAME_CA, + "Set Chain Name Success", + "SSL/TSL chain name:", + &https_ca_name + ); + break; + case 'q': case 'Q': case ESC: + return; + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } + } +} + +// ===============================STATIC API KEY=============================== +void staticAPIKeySettings() { + while(1) { + logMenu(STATIC_API_KEY_MENU); + switch(getSelectedOption()) { + case 'a': case 'A': + setYesNoFields(STATIC_API_KEY_AVAIL, &static_api_key_enabled); + break; + case 'b': case 'B': + setStringFields( + STATIC_API_KEY_VALUE, + "Set Static API Key Success", + "Your static API key:", + &static_api_key_value + ); + break; + case 'q': case 'Q': case ESC: + return; + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } + } +} +// ===============================DEFAULT DATASET=============================== +void defaultDatasetSettings() { + while(1) { + logMenu(DEFAULT_DATASET_MENU); + switch(getSelectedOption()) { + case 'a': case 'A': + setYesNoFields(DEFAULT_DATASET_AVAIL, &default_dataset_enabled); + break; + case 'b': case 'B': + setStringFields( + DEFAULT_DATASET_NAME_MENU, + "Set Default Dataset Name", + "Your default dataset name:", + &default_dataset_name + ); + break; + case 'q': case 'Q': case ESC: + return; + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } + } +} + +// ===============================API=============================== +void logAPIMenu() { + fflush(stdout); + system("clear"); + printf("%s", API_MENU); + printf(API_INDEX_DOC, api_index_doc_enabled?ENABLED:DISABLED); + printf(API_INDEX_STATS, api_index_stats_enabled?ENABLED:DISABLED); + printf(API_INDEX_HEALTHY, api_index_healthy_enabled?ENABLED:DISABLED); + printf(API_INFERENCE_COMP, api_inference_comp_enabled?ENABLED:DISABLED); + printf(API_INFERENCE_RAG, api_inference_rag_enabled?ENABLED:DISABLED); + printf(API_TOKEN, api_token_enabled?ENABLED:DISABLED); + printf(API_EMBEDDING_CALC, api_embedding_calc_enabled?ENABLED:DISABLED); + printf(API_EMBEDDING_DS, api_embedding_ds_enabled?ENABLED:DISABLED); + printf(API_VERSION, api_version_enabled?ENABLED:DISABLED); + printf("%s", API_MENU_END); +} +void apiSettings() { + while(1) { + logAPIMenu(); + switch(getSelectedOption()) { + case 'a': case 'A': + api_index_doc_enabled = !api_index_doc_enabled; break; + case 'b': case 'B': + api_index_stats_enabled = !api_index_stats_enabled; break; + case 'c': case 'C': + api_index_healthy_enabled = !api_index_healthy_enabled; break; + case 'd': case 'D': + api_inference_comp_enabled = !api_inference_comp_enabled; break; + case 'e': case 'E': + api_inference_rag_enabled = !api_inference_rag_enabled; break; + case 'f': case 'F': + api_token_enabled = !api_token_enabled; break; + case 'g': case 'G': + api_embedding_calc_enabled = !api_embedding_calc_enabled; break; + case 'h': case 'H': + api_embedding_ds_enabled = !api_embedding_ds_enabled; break; + case 'i': case 'I': + api_version_enabled = !api_version_enabled; break; + case 'q': case 'Q': case ESC: + return; + case 'y': case 'Y': + api_index_doc_enabled = 1; + api_index_stats_enabled = 1; + api_index_healthy_enabled = 1; + api_inference_comp_enabled = 1; + api_inference_rag_enabled = 1; + api_token_enabled = 1; + api_embedding_calc_enabled = 1; + api_embedding_ds_enabled = 1; + api_version_enabled = 1; + break; + case 'z': case 'Z': + api_index_doc_enabled = 0; + api_index_stats_enabled = 0; + api_index_healthy_enabled = 0; + api_inference_comp_enabled = 0; + api_inference_rag_enabled = 0; + api_token_enabled = 0; + api_embedding_calc_enabled = 0; + api_embedding_ds_enabled = 0; + api_version_enabled = 0; + break; + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } + } +} +// ===============================SAVE SETTINGS=============================== +void buildSettings() { + while(1) { + logMenu(SAVE_AND_BUILD_MENU); + switch(getSelectedOption()) { + case 'a': case 'A': + saveSettings(1); + break; + case 'b': case 'B': + saveDefaultSettings(); + break; + case 'c': case 'C': + saveSettings(0); + system("sh setup/setup.sh"); + break; + case 'q': case 'Q': case ESC: + return; + case '\n': break; + default: logMessage(INVALID_OPTION); break; + } + } +} + +void saveSettings(int show_message) { + // CONFIG + FILE* f = fopen("setup/config.h", "w"); + fprintf(f, CONFIG_H_FILE, + inference_model_name, + inference_model_url, + inference_cpu_cores, + inference_thread_counts, + embedding_cpu_cores, + embedding_thread_counts, + allow_origin_name, + https_enabled, + https_cert_path_host, + https_cert_path_container, + https_cert_name, + https_privkey_name, + https_ca_name, + ( + strcmp(app_expose_port, "http") == 0 ? "80" : + strcmp(app_expose_port, "https") == 0 ? "443" : + app_expose_port + ), + plugin_enabled, + system_instruction, + static_api_key_enabled, + static_api_key_value, + default_dataset_enabled, + default_dataset_name, + api_index_doc_enabled, + api_index_stats_enabled, + api_index_healthy_enabled, + api_inference_comp_enabled, + api_inference_rag_enabled, + api_token_enabled, + api_embedding_calc_enabled, + api_embedding_ds_enabled, + api_version_enabled, + dev_mode_enabled + ); + fclose(f); + + // ENV + f = fopen(".env", "w"); + fprintf(f, ENV_FILE, + app_expose_port, + inference_cpu_cores, + inference_thread_counts, + embedding_cpu_cores, + embedding_thread_counts, + inference_model_name + ); + fclose(f); + + // DOCKER-COMPOSE + f = fopen("docker-compose-adv.yaml", "w"); + // check if should bind secret volume to host machine + int secret_bind_check = + https_enabled && + strcmp(https_cert_path_host, "*") != 0 && + strcmp(https_cert_path_container, "*") != 0; + char *docker_compose_ssl_str; + if(secret_bind_check) { + asprintf( + &docker_compose_ssl_str, COMPOSE_FILE_SSL_INFO, + https_cert_path_host, https_cert_name, https_cert_path_container, https_cert_name, + https_cert_path_host, https_privkey_name, https_cert_path_container, https_privkey_name, + https_cert_path_host, https_ca_name, https_cert_path_container, https_ca_name + ); + } + // check if should bind any volume to host machine + int volume_bind_check = dev_mode_enabled || secret_bind_check; + + fprintf(f, DOCKER_COMPOSE_FILE, + volume_bind_check ? COMPOSE_FILE_VOLUME_SECTION : "", + dev_mode_enabled ? COMPOSE_FILE_DEV_MODE : "", + secret_bind_check ? docker_compose_ssl_str : "" + ); + + if(secret_bind_check) free(docker_compose_ssl_str); + fclose(f); + + // DOCKERFILE + f = fopen("setup/Dockerfile", "w"); + fprintf(f, DOCKERFILE, + ( + api_index_healthy_enabled ? + DOCKERFILE_ENABLE_HEALTHY_CHECK : "" + ), + ( + dev_mode_enabled ? + DOCKERFILE_DEV_COMMAND : DOCKERFILE_PROD_COMMAND + ), + ( + dev_mode_enabled ? + DOCKERFILE_DEV_ENTRYPOINT : DOCKERFILE_PROD_ENTRYPOINT + ) + ); + fclose(f); + + // SHELL + f = fopen("setup/setup.sh", "w"); + fprintf(f, SHELL_FILE, + inference_model_name, + inference_model_url, + EMBEDDING_MODEL_NAME, + EMBEDDING_MODEL_URL + ); + fclose(f); + + if(show_message) logMessage(SAVE_CONFIG_SUCCESS); +} + +void saveDefaultSettings() { + inference_model_name = DEFAULT_INFERENCE_MODEL_NAME; + inference_model_url = DEFAULT_INFERENCE_MODEL_URL; + inference_cpu_cores = DEFAULT_INFERENCE_CPU_CORES; + inference_thread_counts = DEFAULT_INFERENCE_THREAD_COUNTS; + embedding_cpu_cores = DEFAULT_EMBEDDING_CPU_CORES; + embedding_thread_counts = DEFAULT_EMBEDDING_THREAD_COUNTS; + allow_origin_name = DEFAULT_ALLOW_ORIGIN_NAME; + https_enabled = DEFAULT_HTTPS_ENABLED; + https_cert_path_host = DEFAULT_HTTPS_CERT_PATH_HOST; + https_cert_path_container = DEFAULT_HTTPS_CERT_PATH_CONTAINER; + https_cert_name = DEFAULT_HTTPS_CERT_NAME; + https_privkey_name = DEFAULT_HTTPS_PRIVKEY_NAME; + https_ca_name = DEFAULT_HTTPS_CA_NAME; + app_expose_port = DEFAULT_APP_EXPOSE_PORT; + plugin_enabled = DEFAULT_PLUGIN_ENABLED; + system_instruction = DEFAULT_SYSTEM_INSTRUCTION; + static_api_key_enabled = DEFAULT_STATIC_API_KEY_ENABLED; + static_api_key_value = DEFAULT_STATIC_API_KEY; + default_dataset_enabled = DEFAULT_DEFAULT_DATASET_ENABLED; + default_dataset_name = DEFAULT_DEFAULT_DATASET_NAME; + api_index_doc_enabled = DEFAULT_API_INDEX_DOC_ENABLED; + api_index_stats_enabled = DEFAULT_API_INDEX_STATS_ENABLED; + api_index_healthy_enabled = DEFAULT_API_INDEX_HEALTHY_ENABLED; + api_inference_comp_enabled = DEFAULT_API_INFERENCE_COMP_ENABLED; + api_inference_rag_enabled = DEFAULT_API_INFERENCE_RAG_ENABLED; + api_token_enabled = DEFAULT_API_TOKEN_ENABLED; + api_embedding_calc_enabled = DEFAULT_API_EMBEDDING_CALC_ENABLED; + api_embedding_ds_enabled = DEFAULT_API_EMBEDDING_DS_ENABLED; + api_version_enabled = DEFAULT_API_VERSION_ENABLED; + dev_mode_enabled = DEFAULT_DEV_MODE_ENABLED; + saveSettings(1); +} \ No newline at end of file diff --git a/setup/setup.sh b/setup/setup.sh new file mode 100644 index 0000000..07bc8aa --- /dev/null +++ b/setup/setup.sh @@ -0,0 +1,8 @@ +LANGUAGE_MODEL_NAME=Phi3-mini-4k-instruct-Q4.gguf +LANGUAGE_MODEL_URL=https://huggingface.co/aisuko/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi3-mini-4k-instruct-Q4.gguf +EMBEDDING_MODEL_NAME=all-MiniLM-L6-v2-Q4_K_M-v2.gguf +EMBEDDING_MODEL_URL=https://huggingface.co/aisuko/all-MiniLM-L6-v2-gguf/resolve/main/all-MiniLM-L6-v2-Q4_K_M-v2.gguf +mkdir -p volumes/models && [ -f volumes/models/$LANGUAGE_MODEL_NAME ] || wget -O volumes/models/$LANGUAGE_MODEL_NAME $LANGUAGE_MODEL_URL +mkdir -p volumes/models && [ -f volumes/models/$EMBEDDING_MODEL_NAME ] || wget -O volumes/models/$EMBEDDING_MODEL_NAME $EMBEDDING_MODEL_URL +env $(cat .env) docker compose -f docker-compose-adv.yaml up --build -d +docker container logs -f voyager \ No newline at end of file diff --git a/setup/setup_types.h b/setup/setup_types.h new file mode 100644 index 0000000..44ab9de --- /dev/null +++ b/setup/setup_types.h @@ -0,0 +1,609 @@ +#ifndef SETUP_TYPES_H +#define SETUP_TYPES_H + +#include +#include +#include +#include "config.h" +#include "default_config.h" + +#define ESC 27 + +#define MESSAGE_MENU \ + "================================\n"\ + "\n"\ + " %s\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Press Enter to Continue..." + +#define INVALID_OPTION "This Option is Invalid!" + +#define MENU_TEXT \ + "================================\n"\ + "\n"\ + " Welcome to Voyager APIs!\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "a. Config Inference Engine\n"\ + "b. Config Embedding Engine\n"\ + "c. Set Allow Origin Name\n"\ + "d. Network Related Features\n"\ + "e. Set Plugin Availability\n"\ + "f. Set System Instruction\n"\ + "g. Set Default Dataset Features\n"\ + "h. Set Static API-Key Features\n"\ + "i. Set APIs Availability\n"\ + "j. Toggle Developer Mode\n"\ + "k. Save & Build\n"\ + "q. Quit\n"\ + "\n"\ + "Please enter your selection: " + +// ===============================INFERENCE ENGINE=============================== +#define INFERENCE_ENG_MENU \ + "================================\n"\ + "\n"\ + " Inference Engine Settings\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "a. Set Model\n"\ + "b. Set CPU Cores\n"\ + "c. Set Thread Counts\n"\ + "q. Go back\n"\ + "\n"\ + "Please enter your selection: " + +#define INFERENCE_ENG_MODEL \ + "================================\n"\ + "\n"\ + " Models for Inference Engine\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "a. (Approx 2.2GB, Slow, High Response Quality) Phi3-mini-4k-instruct-Q4.gguf\n"\ + "b. (Approx 270MB, Fast, Low Response Quality) ft-smollm-135M-instruct-on-hf-ultrafeedback-f16.gguf\n"\ + "q. Go back\n"\ + "\n"\ + "Please enter your selection: " + +#define MODEL_PHI 'a' +#define MODEL_PHI_NAME "Phi3-mini-4k-instruct-Q4.gguf" +#define MODEL_PHI_URL "https://huggingface.co/aisuko/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi3-mini-4k-instruct-Q4.gguf" + +#define MODEL_SMOLLM 'b' +#define MODEL_SMOLLM_NAME "ft-smollm-135M-instruct-on-hf-ultrafeedback-f16.gguf" +#define MODEL_SMOLLM_URL "https://huggingface.co/aisuko/ft-smollm-135M-instruct-on-hf-ultrafeedback-gguf/resolve/main/ft-smollm-135M-instruct-on-hf-ultrafeedback-f16.gguf" + +#define INFERENCE_ENG_CPU \ + "================================\n"\ + "\n"\ + " CPU Cores of Inference Engine\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please enter the number of CPU cores: " + +#define INFERENCE_ENG_THREAD \ + "===================================\n"\ + "\n"\ + " Thread Counts of Inference Engine\n"\ + "\n"\ + "===================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please enter the number of threads: " + +// ===============================EMBEDDING ENGINE=============================== +#define EMBEDDING_ENG_MENU \ + "================================\n"\ + "\n"\ + " Embedding Engine Settings\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "a. Set CPU Cores\n"\ + "b. Set Thread Counts\n"\ + "q. Go back\n"\ + "\n"\ + "Please enter your selection: " + +#define EMBEDDING_MODEL_NAME "all-MiniLM-L6-v2-Q4_K_M-v2.gguf" +#define EMBEDDING_MODEL_URL "https://huggingface.co/aisuko/all-MiniLM-L6-v2-gguf/resolve/main/all-MiniLM-L6-v2-Q4_K_M-v2.gguf" + +#define EMBEDDING_ENG_CPU \ + "================================\n"\ + "\n"\ + " CPU Cores of Embedding Engine\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please enter the number of CPU cores: " + +#define EMBEDDING_ENG_THREAD \ + "===================================\n"\ + "\n"\ + " Thread Counts of Embedding Engine\n"\ + "\n"\ + "===================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please enter the number of threads: " + +// ===============================ALLOW ORIGIN=============================== +#define ALLOW_ORIGIN_MENU \ + "================================\n"\ + "\n"\ + " Set Allowed Origin Name\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Max length is 100 characters.\n"\ + "Don't forget the 'https://' prefix and leave no '\\' on the end.\n"\ + "Enter 'q' to go back, or '*' to allow all origins.\n\n"\ + "Please enter the allowed origin name: " + +#define ALLOW_ORIGIN_SUCCESS_MSG "Set Allow Origin Success" + +// ===============================HTTPS=============================== +#define HTTPS_MENU \ + "================================\n"\ + "\n"\ + " Set Network Features\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "a. Enable or Disable Secure Connection (https)\n"\ + "b. Set SSL/TLS Certificate Store Path\n"\ + "c. Set SSL/TLS Certificate Names\n"\ + "d. Set Application Expose Port\n"\ + "q. Go back\n"\ + "\n"\ + "Please enter your selection: " + +#define HTTPS_AVAIL \ + "================================\n"\ + "\n"\ + " Enable or Disable Https\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Do you want to enable https features? (Y/N): " + +#define HTTPS_CERT_LOC \ + "====================================\n"\ + "\n"\ + " Set Path of SSL/TLS Certificates\n"\ + "\n"\ + "====================================\n"\ + "\n"\ + "a. Path on Host Machine\n"\ + "b. Path in Container\n"\ + "c. Path of Both\n"\ + "q. Go back\n"\ + "\n"\ + "Please enter your selection: " + +#define HTTPS_CERT_LOC_HOST \ + "================================\n"\ + "\n"\ + " Path on Host Machine\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please enter your path of SSL/TLS Certificate path on local machine:\n" + +#define HTTPS_CERT_LOC_CONTAINER \ + "================================\n"\ + "\n"\ + " Path in Container\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please enter your path of SSL/TLS Certificate path in container:\n" + +#define HTTPS_CERT_LOC_BOTH \ + "================================\n"\ + "\n"\ + " Path for Both\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please enter your path of SSL/TLS Certificate for both host machine and container:\n" + +#define HTTPS_CERT_NAME_MENU \ + "====================================\n"\ + "\n"\ + " Set Name of SSL/TLS Certificates\n"\ + "\n"\ + "====================================\n"\ + "\n"\ + "a. Name of Certificate...(default cert.pem)\n"\ + "b. Name of Private Key...(default privkey.pem)\n"\ + "c. Name of Chain.........(default chain.pem)\n"\ + "q. Go back\n"\ + "\n"\ + "Please enter your selection: " + +#define HTTPS_CERT_NAME_CERT \ + "====================================\n"\ + "\n"\ + " Set Name of SSL/TLS Certificate\n"\ + "\n"\ + "====================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please enter your name of SSL/TLS certificate:\n" + +#define HTTPS_CERT_NAME_KEY \ + "====================================\n"\ + "\n"\ + " Set Name of SSL/TLS Privae Key\n"\ + "\n"\ + "====================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please enter your name of SSL/TLS private key:\n" + +#define HTTPS_CERT_NAME_CA \ + "====================================\n"\ + "\n"\ + " Set Name of SSL/TLS Chain\n"\ + "\n"\ + "====================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please enter your name of SSL/TLS chain:\n" + +#define APP_OPEN_PORT_MENU \ + "================================\n"\ + "\n"\ + " App Expose Port\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Please enter a port number.\n"\ + "String \"http\" equals \"80\" and \"https\" equals \"443\".\n"\ + "Enter 'q' to go back\n\n"\ + "Please enter your the port you want to expose: " + +// ===============================PLUGIN=============================== +#define PLUGIN_MENU \ + "================================\n"\ + "\n"\ + " Enable or Disable Plugins\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Remember to write your own plugins in /tools/plugin.js\nif you want to enable it.\n\n"\ + "Enter 'q' to go back\n\n"\ + "Do you want to enable plugins? (Y/N): " + +// ===============================SYSTEM INSTRUCTION=============================== +#define SYSTEM_INSTRUCTION_MENU \ + "================================\n"\ + "\n"\ + " Set System Instruction\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Set the system instruction for AI to inference, for example:\n\n"\ + "You are a helpful assistant who provide correct information to users.\n\n"\ + "Max length is 500 characters.\n"\ + "Enter 'q' to go back.\n\n"\ + "Please enter the system instruction:\n" + +// ===============================STATIC API KEY=============================== +#define STATIC_API_KEY_MENU \ + "================================\n"\ + "\n"\ + " Static API Key Features\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "You can choose to set your static API key name manually in your enviromnet\n"\ + "Where the name environment variable name is STATIC_API_KEY.\n"\ + "\n"\ + "a. Enable or disable static API key\n"\ + "b. Set static API key value\n"\ + "q. Go back\n"\ + "\n"\ + "Please enter your selection: " + +#define STATIC_API_KEY_AVAIL \ + "==================================\n"\ + "\n"\ + " Enable or Disable Static API Key\n"\ + "\n"\ + "==================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Do you want to enable static API Key? (Y/N): " + +#define STATIC_API_KEY_VALUE \ + "================================\n"\ + "\n"\ + " Set Static API Key\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please input the value of your static API key:\n" + +// ===============================DEFAULT DATASET=============================== +#define DEFAULT_DATASET_MENU \ + "================================\n"\ + "\n"\ + " Default Dataset Features\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "a. Enable or disable default dataset\n"\ + "b. Set default dataset name\n"\ + "q. Go back\n"\ + "\n"\ + "Please enter your selection: " + +#define DEFAULT_DATASET_AVAIL \ + "===================================\n"\ + "\n"\ + " Enable or Disable Default Dataset\n"\ + "\n"\ + "===================================\n"\ + "\n"\ + "Please implement your own dataset in /tools/plugin.js if you want to enable it\n"\ + "Enter 'q' to go back\n\n"\ + "Do you want to enable load default dataset? (Y/N): " + +#define DEFAULT_DATASET_NAME_MENU \ + "================================\n"\ + "\n"\ + " Set Default Dataset Name\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Enter 'q' to go back\n"\ + "Please input the name of the default dataset:\n" +// ===============================APIs=============================== +#define API_MENU \ + "================================\n"\ + "\n"\ + " APIs\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "Select the letter to toggle corresponding API.\n\n" + +#define API_INDEX_DOC \ + "a. %s Docs.................(/)\n" + +#define API_INDEX_STATS \ + "b. %s Stats................(/stats)\n" + +#define API_INDEX_HEALTHY \ + "c. %s Healthy..............(/healthy)\n" + +#define API_INFERENCE_COMP \ + "d. %s Inference............(/v1/chat/completions)\n" + +#define API_INFERENCE_RAG \ + "e. %s RAG Inference........(/v1/chat/rag-completions)\n" + +#define API_TOKEN \ + "f. %s API Key..............(/v1/token/api-key)\n" + +#define API_EMBEDDING_CALC \ + "g. %s Calcualte Embedding..(/v1/embeddings)\n" + +#define API_EMBEDDING_DS \ + "h. %s Upload Dataset.......(/v1/embeddings/dataset)\n" + +#define API_VERSION \ + "i. %s Version..............(/v1/version)\n" + +#define API_MENU_END \ + "y. Enable All\n"\ + "z. Disable All\n"\ + "q. Go back\n"\ + "\n"\ + "Please enter your selection: " + +#define ENABLED " (Enabled)" +#define DISABLED "(Disabled)" + +// ===============================DEV MODE=============================== +#define DEV_MODE_MENU \ + "================================\n"\ + "\n"\ + " Developer Mode\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "The developer mode can auto restart your server if you changed any file.\n"\ + "This requires you installed dependencies in 'package.json',\notherwise the server will fail to start.\n\n"\ + "You can choose to save settings from 'Save & Build' menu \nand install use your prefer package manager.\n\n"\ + "Enter 'q' to go back\n\n"\ + "Do you want to enable developer mode? (Y/N): " + +// ===============================CONFIGS=============================== +#define SAVE_CONFIG_SUCCESS "Successfully Saved Config" + +#define SAVE_AND_BUILD_MENU \ + "================================\n"\ + "\n"\ + " Save And Build Server\n"\ + "\n"\ + "================================\n"\ + "\n"\ + "a. Only save configs\n"\ + "b. Load default configs\n"\ + "c. Build and start the server\n"\ + "q. Go back\n"\ + "\n"\ + "Please enter your selection: " + +#define CONFIG_H_FILE \ +"#include \"setup_types.h\"\n"\ +"\n"\ +"#ifndef SETUP_CONFIG_H\n"\ +"#define SETUP_CONFIG_H\n"\ +"\n"\ +"#define INFERENCE_MODEL_NAME \"%s\"\n"\ +"#define INFERENCE_MODEL_URL \"%s\"\n"\ +"#define INFERENCE_CPU_CORES %.2f\n"\ +"#define INFERENCE_THREAD_COUNTS %.2f\n"\ +"\n"\ +"#define EMBEDDING_CPU_CORES %.2f\n"\ +"#define EMBEDDING_THREAD_COUNTS %.2f\n"\ +"\n"\ +"#define ALLOW_ORIGIN_NAME \"%s\"\n"\ +"\n"\ +"#define HTTPS_ENABLED %d\n"\ +"#define HTTPS_CERT_PATH_HOST \"%s\"\n"\ +"#define HTTPS_CERT_PATH_CONTAINER \"%s\"\n"\ +"#define HTTPS_CERT_NAME \"%s\"\n"\ +"#define HTTPS_PRIVKEY_NAME \"%s\"\n"\ +"#define HTTPS_CA_NAME \"%s\"\n"\ +"#define APP_EXPOSE_PORT \"%s\"\n"\ +"\n"\ +"#define PLUGIN_ENABLED %d\n"\ +"\n"\ +"#define SYSTEM_INSTRUCTION \"%s\"\n"\ +"\n"\ +"#define STATIC_API_KEY_ENABLED %d\n"\ +"#define STATIC_API_KEY \"%s\"\n"\ +"\n"\ +"#define DEFAULT_DATASET_ENABLED %d\n"\ +"#define DEFAULT_DATASET_NAME \"%s\"\n"\ +"\n"\ +"#define API_INDEX_DOC_ENABLED %d\n"\ +"#define API_INDEX_STATS_ENABLED %d\n"\ +"#define API_INDEX_HEALTHY_ENABLED %d\n"\ +"#define API_INFERENCE_COMP_ENABLED %d\n"\ +"#define API_INFERENCE_RAG_ENABLED %d\n"\ +"#define API_TOKEN_ENABLED %d\n"\ +"#define API_EMBEDDING_CALC_ENABLED %d\n"\ +"#define API_EMBEDDING_DS_ENABLED %d\n"\ +"#define API_VERSION_ENABLED %d\n"\ +"\n"\ +"#define DEV_MODE_ENABLED %d\n"\ +"\n"\ +"#endif" + +#define DOCKER_COMPOSE_FILE \ +"services:\n"\ +" llamacpp:\n"\ +" container_name: ${INFERENCE_ENG}\n"\ +" image: gclub/llama.cpp:${INFERENCE_ENG_VERSION}\n"\ +" restart: always\n"\ +" deploy:\n"\ +" resources:\n"\ +" reservations:\n"\ +" cpus: \"${NUM_CPU_CORES}\"\n"\ +" volumes:\n"\ +" - \"${DOCKER_VOLUME_DIRECTORY:-.}/${MODEL_SAVE_PATH}:/models\"\n"\ +" expose:\n"\ +" - ${ENG_ACCESS_PORT}\n"\ +" ports:\n"\ +" - ${INFERENCE_ENG_PORT}:${ENG_ACCESS_PORT}\n"\ +" command: [\"-m\", \"models/${LANGUAGE_MODEL_NAME}\",\"-t\",\"${NUM_THREADS_COUNT}\",\"-c\",\"8192\"]\n"\ +"\n"\ +" embedding_eng:\n"\ +" container_name: ${EMBEDDING_ENG}\n"\ +" image: gclub/llama.cpp:${INFERENCE_ENG_VERSION}\n"\ +" restart: always\n"\ +" deploy:\n"\ +" resources:\n"\ +" reservations:\n"\ +" cpus: \"${NUM_CPU_CORES_EMBEDDING}\"\n"\ +" volumes:\n"\ +" - \"${DOCKER_VOLUME_DIRECTORY:-.}/${MODEL_SAVE_PATH}:/models\"\n"\ +" expose:\n"\ +" - ${ENG_ACCESS_PORT}\n"\ +" ports:\n"\ +" - ${EMBEDDING_ENG_PORT}:${ENG_ACCESS_PORT}\n"\ +" command: [\"-m\", \"models/${EMBEDDING_MODEL_NAME}\",\"--embeddings\",\"--pooling\",\"mean\",\"-t\",\"${NUM_THREAD_COUNTS_EMBEDDING}\",\"-c\",\"512\"]\n"\ +"\n"\ +" voyager:\n"\ +" container_name: voyager\n"\ +" restart: always\n"\ +" build:\n"\ +" dockerfile: setup/Dockerfile\n"\ +" context: .\n"\ +"%s%s%s"\ +" expose:\n"\ +" - ${APP_EXPOSE_PORT}\n"\ +" ports:\n"\ +" - ${APP_EXPOSE_PORT}:${APP_PORT}\n"\ +" depends_on:\n"\ +" - llamacpp\n"\ +" - embedding_eng\n"\ + +#define COMPOSE_FILE_VOLUME_SECTION " volumes:\n" +#define COMPOSE_FILE_DEV_MODE " - .:/app\n" +#define COMPOSE_FILE_SSL_INFO \ +" - %s/%s:%s/%s\n"\ +" - %s/%s:%s/%s\n"\ +" - %s/%s:%s/%s\n" + +#define ENV_FILE \ +"APP_PORT=8000\n"\ +"APP_EXPOSE_PORT=%s\n"\ +"ENG_ACCESS_PORT=8080\n"\ +"MODEL_SAVE_PATH=volumes/models\n"\ +"INFERENCE_ENG=llamacpp\n"\ +"INFERENCE_ENG_PORT=8080\n"\ +"INFERENCE_ENG_VERSION=server--b1-27d4b7c\n"\ +"NUM_CPU_CORES=%.2f\n"\ +"NUM_THREADS_COUNT=%.2f\n"\ +"EMBEDDING_ENG=embedding_eng\n"\ +"EMBEDDING_ENG_PORT=8081\n"\ +"NUM_CPU_CORES_EMBEDDING=%.2f\n"\ +"NUM_THREAD_COUNTS_EMBEDDING=%.2f\n"\ +"LANGUAGE_MODEL_NAME=%s\n"\ +"EMBEDDING_MODEL_NAME=all-MiniLM-L6-v2-Q4_K_M-v2.gguf\n"\ + +#define DOCKERFILE \ +"FROM node:20.15.1-slim\n"\ +"WORKDIR /app\n"\ +"COPY . .\n"\ +"\n"\ +"%s"\ +"%s"\ +"%s"\ + +#define DOCKERFILE_ENABLE_HEALTHY_CHECK \ +"HEALTHCHECK --interval=300s --timeout=30s --start-period=5s --retries=3 CMD [ \"node\", \"healthy-check.js\" ]\n" + +#define DOCKERFILE_DEV_COMMAND \ +"RUN npm install -g nodemon\n" + +#define DOCKERFILE_PROD_COMMAND \ +"RUN npm install -g pnpm && pnpm install\n" + +#define DOCKERFILE_DEV_ENTRYPOINT \ +"ENTRYPOINT [ \"npm\", \"run\", \"dev\" ]" + +#define DOCKERFILE_PROD_ENTRYPOINT \ +"ENTRYPOINT [ \"npm\", \"start\" ]" + +#define SHELL_FILE \ +"LANGUAGE_MODEL_NAME=%s\n"\ +"LANGUAGE_MODEL_URL=%s\n"\ +"EMBEDDING_MODEL_NAME=%s\n"\ +"EMBEDDING_MODEL_URL=%s\n"\ +"mkdir -p volumes/models && [ -f volumes/models/$LANGUAGE_MODEL_NAME ] || wget -O volumes/models/$LANGUAGE_MODEL_NAME $LANGUAGE_MODEL_URL\n"\ +"mkdir -p volumes/models && [ -f volumes/models/$EMBEDDING_MODEL_NAME ] || wget -O volumes/models/$EMBEDDING_MODEL_NAME $EMBEDDING_MODEL_URL\n"\ +"env $(cat .env) docker compose -f docker-compose-adv.yaml up --build -d\n"\ +"docker container logs -f voyager" + +#endif \ No newline at end of file