Small updates to build and start.sh

.env

@@ -1,7 +1,9 @@
 MODEL_DIR=/media/teamgroup/models
-MODEL=EverythingLM-13B-V3-16K-GPTQ
+MODEL=Mistral-7B-instruct-exl2
 EXTENSIONS_DIR=/home/me/git_repos/extensions/
 CHARACTERS_DIR=/media/teamgroup/characters
 SETTINGS_DIR=/media/teamgroup/settings
-COMMITHASH=7a3ca2c68f1ca49ac4e4b62f016718556fd3805c
-LOGS_DIR=./logs
+COMMITHASH=32984ea2f0a5a3f6be206e3bcef3e39d18a82006
+LOGS_DIR=./logs
+DATASETS_DIR=/media/teamgroup/loras/datasets
+LORAS_DIR=/media/teamgroup/loras

Dockerfile

@@ -18,12 +18,10 @@ SHELL ["conda", "run", "-n", "textgen", "/bin/bash", "-c"]
 ENV CUDA_DOCKER_ARCH=all
 
 # Installing torch and ninja
-RUN pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
+RUN pip3 install torch==2.1.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
 
 RUN pip3 install ninja packaging
 
-RUN MAX_JOBS=4 pip3 install flash-attn --no-build-isolation 
-
 ARG commithash=.
 
 # Pulling latest text-generation-webui branch
@@ -34,8 +32,8 @@ RUN git clone https://github.com/oobabooga/text-generation-webui.git  \
 # Install all the extension requirements
 RUN bash -c 'for i in text-generation-webui/extensions/*/requirements.txt ; do pip3 install -r $i ; done'
 
-# Prepare cache for faster first time runs NOTE: This is currently broken on master, will re-add when it's fixed: https://github.com/oobabooga/text-generation-webui/pull/3995
-#RUN cd /text-generation-webui/extensions/openai/ && python3 cache_embedding_model.py
+# Prepare cache for faster first time runs
+RUN cd /text-generation-webui/extensions/openai/ && python3 cache_embedding_model.py
 
 RUN conda clean -afy
 

docker-compose.yml

@@ -2,18 +2,22 @@ version: '3.9'
 services:
   text-generation-webui:
     container_name: text-generation-webui
-    restart: always
+    restart: unless-stopped
     volumes:
       - '${MODEL_DIR}:/models'
       - '${EXTENSIONS_DIR}:/extensions'
       - '${CHARACTERS_DIR}:/characters'
       - '${SETTINGS_DIR}:/settings'
       - '${LOGS_DIR}:/logs'
+      - '${LORAS_DIR}:/loras'
+      - '${DATASETS_DIR}:/datasets'
     ports:
       - '7860:7860'
       - '5000:5000'
       - '5001:5001'
     image: 'noneabove1182/text-gen-ui-gpu:latest'
+    environment:
+      - OPENEDAI_EMBEDDING_DEVICE=cuda
     ulimits:
       memlock: -1
     mem_limit: 50gb
@@ -25,14 +29,12 @@ services:
         reservations:
           devices:
             - driver: nvidia
-              device_ids: [ "0" ]
+              count: 1
               capabilities: [ gpu ]
     command:
       [
         "--model",
         "${MODEL}",
-        "--loader",
-        "exllama_hf",
         "--listen",
         "--listen-port",
         "7860",

start.sh

@@ -19,6 +19,15 @@ else
     echo "No loras DIR"
 fi
 
+if [[ -d /datasets ]] ; then
+    echo "Found datasets, linking the directory"
+    rm -r /text-generation-webui/training/datasets
+    ln -s /datasets /text-generation-webui/training/datasets
+else
+    echo "No datasets DIR"
+fi
+
+
 if [[ -d /prompts ]] ; then
     echo "Found prompts, copying repo prompts to local and linking the directory"
     cp -R /text-generation-webui/prompts/. /prompts