Chatbox improve

README.md

@@ -60,30 +60,45 @@ make dev
 ```
 **NOTE:** `make dev` Requires Node.js environment installed, or at least have `node_modules` specified in `package.json` installed on your server. Please see [Local Machine](#local-machine) section.
 
-## Embed your chatbot into website
-To embed a chatbot to your website, simply add  
+## Lint
+To start lint your code, simply run
+```shell
+npm run lint
+```
+
+## APIs
+
+### Docs
+Go to the url of your project, default [http://localhost:8000](http://localhost:8000) if you didn't disabled the `Docs` route, then you can see docs and try it on.  
+See [demo video](#setup-and-api-usage-demo-video).
+
+### Monitor
+This project got monitor build with swagger-stats, when you got this project running, just go to `<Your Server>:<Your Port>/stats`.  
+For example, [http://localhost:8000/stats](http://localhost:8000/stats)
+
+### Chatbox
+> When you set up the project and didn't disabled the `chatbox` API, you can get a quick-setup chatbot with some basic styles on your own website, which calls the `/v1/chat/completions` API for inference.  
+  
+To set it up, simply add  
 ```html
-<script src='http://localhost:8000/chatbox?base_url=http%3A%2F%2Flocalhost%3A8000' defer></script>
+<script src='http://localhost:8000/chatbox' defer></script>
 ```
 into the bottom of your html body element. So easy!  
-Please change the `http://localhost:8000` and the base_url in request query to your real host to make sure it works.  
 
 If you want to hide the real link, in your javascript code you can do  
 ```js
-const chatbox_script = await (await fetch("http://localhost:8000/chatbox?base_url=http%3A%2F%2Flocalhost%3A8000")).blob();
+const chatbox_script = await (await fetch("http://localhost:8000/chatbox")).blob();
 const chatbox_url = URL.createObjectURL(chatbox_script);
 const script_elem = document.createElement('script');
 script_elem.src = chatbox_url;
 document.body.append(script_elem);
 ```
-And remember to use `URL.revokeObjectURL(chatbox_url)` if you don't need it anymore.
-
-## Lint
-To start lint your code, simply run
-```shell
-npm run lint
-```
-
-## Monitor
-This project got monitor build with swagger-stats, when you got this project running, just go to `<Your Server>:<Your Port>/stats`.  
-For example, [http://localhost:8000/stats](http://localhost:8000/stats)
+And remember to use `URL.revokeObjectURL(chatbox_url)` if you don't need it anymore.  
+
+Extra parameters ([request query](https://en.wikipedia.org/wiki/Query_string)) you can add to it are:
+* `base_url`: `String`  
+    > Add this when in production, otherwise the requests won't send to correct route.  
+    > Default `http://localhost:8000`.
+* `max_tokens`: `Integer`  
+    > Add this when you want to limit tokens can be generated, is useful in production.
+    > Default `128`

actions/inference.js

@@ -83,7 +83,12 @@ async function doInference(req_body, callback, isStream) {
             const { value, done } = await reader.read();
             if(done) break;
             const data = value.split("data: ").pop()
-            callback(JSON.parse(data));
+            try {
+                callback(JSON.parse(data));
+            } catch(error) {
+                console.log(error)
+                callback({content: "", stop: true})
+            }
         }
     } else {
         const eng_resp = await post('completion', { body: req_body });

example/demo_ui.py

@@ -0,0 +1,153 @@
+import streamlit as st
+import random
+import string
+import datetime
+import tiktoken
+from openai import OpenAI
+
+RMIT="RMIT"
+
+encoding = tiktoken.get_encoding("r50k_base") 
+
+client = OpenAI(base_url="http://localhost:8080/v1", api_key="not-needed", organization='SelectedModel')
+
+def writehistory(filename,text):
+    with open(filename, 'a', encoding='utf-8') as f:
+        f.write(text)
+        f.write('\n')
+    f.close()
+
+#AVATARS  👷🐦  🥶🌀
+av_us = '🦖'  #"🦖"  #A single emoji, e.g. "🧑‍💻", "🤖", "🦖".
+av_ass = '👷'
+
+# Set the webpage title
+st.set_page_config(
+    page_title=f"Your LocalGPT with 🌟 {RMIT}",
+    page_icon="🌟",
+    layout="wide")
+
+# Create a header element
+mytitle = '# Local AI Solutions and LLM-based Chatbot development 🌟'
+st.markdown(mytitle, unsafe_allow_html=True)
+st.markdown('### Conversation tokens context window')
+# function to generate random alphanumeric sequence for the filename
+def genRANstring(n):
+    """
+    n = int number of char to randomize
+    """
+    N = n
+    res = ''.join(random.choices(string.ascii_uppercase +
+                                string.digits, k=N))
+    return res
+
+# create THE SESSIoN STATES
+if "logfilename" not in st.session_state:
+## Logger file
+    logfile = f'{genRANstring(5)}_log.txt'
+    st.session_state.logfilename = logfile
+    #Write in the history the first 2 sessions
+    writehistory(st.session_state.logfilename,f'{str(datetime.datetime.now())}\n\nYour own LocalGPT with 🌀 {RMIT}\n---\n🧠🫡: You are a helpful assistant.')    
+    writehistory(st.session_state.logfilename,f'🌀: How may I help you today?')
+
+if "repeat" not in st.session_state:
+    st.session_state.repeat = 1.35
+
+if "temperature" not in st.session_state:
+    st.session_state.temperature = 0.1
+
+if "maxlength" not in st.session_state:
+    st.session_state.maxlength = 500
+
+
+# CREATE THE SIDEBAR
+with st.sidebar:
+    st.image('img/RMIT_POS3.png', use_column_width=True)
+    st.session_state.temperature = st.slider('Temperature:', min_value=0.0, max_value=1.0, value=0.1, step=0.02)
+    st.session_state.maxlength = st.slider('Length reply:', min_value=150, max_value=1000, 
+                                           value=500, step=50)
+    st.session_state.repeat = st.slider('Repeat Penalty:', min_value=0.0, max_value=2.0, value=1.35, step=0.01)
+    st.markdown(f"**Logfile**: {st.session_state.logfilename}")
+    btnClear = st.button("Clear History",type="primary", use_container_width=True)
+
+
+# We store the conversation in the session state.
+# This will be used to render the chat conversation.
+# We initialize it with the first message we want to be greeted with. 
+#Note that the first 3 messages will never be used for the genration, they are only for the Chat interface
+if "messages" not in st.session_state:
+    st.session_state.messages = [
+        {"role": "system", "content": "You are assistant, a helpful assistant. You reply only to the user questions. You always reply in the language of the instructions.",},
+        {"role": "user", "content": "Hi, I am P000252SE."},
+        {"role": "assistant", "content": "Hi there, I am assistant, how may I help you today?"}
+    ]
+# we define the function to clear from the screen the conversation history
+def clearHistory():
+    st.session_state.messages = [
+        {"role": "system", "content": "You are assistant, a helpful assistant. You reply only to the user questions. You always reply in the language of the instructions.",},
+        {"role": "user", "content": "Hi, I am P000252SE."},
+        {"role": "assistant", "content": "Hi there, I am assistant, how may I help you today?"}
+    ]
+if btnClear:
+      clearHistory()  
+
+
+# We loop through each message in the session state and render it as # a chat message.
+for message in st.session_state.messages[1:]:
+    if message["role"] == "user":
+        with st.chat_message(message["role"],avatar=av_us):
+            st.markdown(message["content"])
+    else:
+        with st.chat_message(message["role"],avatar=av_ass):
+            st.markdown(message["content"])
+
+# We take questions/instructions from the chat input to pass to the LLM
+if user_prompt := st.chat_input("Your message here. Shift+Enter to add a new line", key="user_input"):
+
+    # Add our input to the session state
+    st.session_state.messages.append(
+        {"role": "user", "content": user_prompt}
+    )
+
+    # Add our input to the chat window
+    with st.chat_message("user", avatar=av_us):
+        st.markdown(user_prompt)
+        writehistory(st.session_state.logfilename,f'👷: {user_prompt}')
+
+
+    with st.chat_message("assistant",avatar=av_ass):
+        message_placeholder = st.empty()
+        with st.spinner("Inferencing..."):
+            response = ''
+            conv_messages = []
+            conv_messages.append(st.session_state.messages[-1])
+            full_response = ""
+
+            completion = client.chat.completions.create(
+                model="local-model",
+                messages=conv_messages,  #st.session_state.messages if you want to keep previous messages,
+                temperature=st.session_state.temperature,
+                frequency_penalty  = st.session_state.repeat,
+                stop=['<|im_end|>','</s>'],
+                max_tokens=st.session_state.maxlength,
+                stream=True,
+            )
+            for chunk in completion:
+                if chunk.choices[0].delta.content:
+                    full_response += chunk.choices[0].delta.content
+                    message_placeholder.markdown(full_response + "🌟")
+
+            toregister = full_response + f"""
+```
+
+prompt tokens: {len(encoding.encode(st.session_state.messages[-1]['content']))}
+generated tokens: {len(encoding.encode(full_response))}
+```"""
+            message_placeholder.markdown(toregister)
+            writehistory(st.session_state.logfilename,f'🌟: {toregister}\n\n---\n\n') 
+
+
+    # Add the response to the session state
+    st.session_state.messages.append(
+        {"role": "assistant", "content": toregister}
+    )

routes/index.js

@@ -36,8 +36,9 @@ function indexRoute() {
 
     if(isRouteEnabled("index", "chatbox")) {
         router.get('/chatbox', (req, res)=>{
+            const { base_url, max_tokens } = req.query;
             res.setHeader("Content-Type", "application/json; charset=utf-8")
-            res.send(generateScript(req.query.base_url));
+            res.send(generateScript(base_url, max_tokens));
         })
     }
 

tools/web_embed.js

@@ -2,7 +2,7 @@
  * Generates the script with base_url, to simply embed a chatbot in web page
  * @param {String} base_url The url to send request, default `http://localhost:8000`
  */
-export function generateScript(base_url = 'http://localhost:8000') {
+export function generateScript(base_url = 'http://localhost:8000', max_tokens = 128) {
     return `
 (function() {
     'use strict';
@@ -105,6 +105,7 @@ const styles = \`
     border-radius: 20px;
     border: 1px solid gray;
     padding: 0px 40px 0px 10px;
+    box-sizing: border-box;
 }
 
 
@@ -176,6 +177,7 @@ const styles = \`
     top: 40px;
     display: block;
     overflow-y: auto;
+    overflow-x: hidden;
     padding: 20px;
     box-sizing: border-box;
 }
@@ -188,6 +190,12 @@ const styles = \`
     text-align: center;
     text-decoration: none;
     display: block;
+    transition-duration: .3s;
+}
+
+.chatbox > .expanded-page > .conversations > .power-by:hover {
+    transform: scale(1.2);
+    font-weight: bold;
 }
 
 .chatbox > .expanded-page > .conversations > .bubble {
@@ -199,6 +207,8 @@ const styles = \`
     margin: auto;
     font-size: 17px;
     margin-bottom: 10px;
+    word-wrap: break-word;
+    max-width: calc(100% - 40px);
 }
 
 .chatbox > .expanded-page > .conversations > .bubble.empty {
@@ -291,7 +301,9 @@ const styles = \`
     function createElement(tagName, className, textContent) {
         const elem = document.createElement(tagName);
         elem.className = className;
-        elem.textContent = textContent;
+        textContent.split("\\n").forEach(content=>{
+            elem.append(content, document.createElement("br"))
+        })
         return elem;
     }
     
@@ -313,22 +325,37 @@ const styles = \`
                     {role: 'system', content: "You are a helpful assistant solves problem"},
                     {role: 'user', content: message}
                 ],
-                stream: true
+                stream: true,
+                max_tokens: ${max_tokens}
             })
         });
         if(resp.ok) {
             const reader = resp.body.pipeThrough(new TextDecoderStream()).getReader();
-            let response = ''
+            let response = '', started = false;
             while(true) {
                 const { value, done } = await reader.read();
+                if(!started) {
+                    started = true;
+                    pending_conversation.textContent = '';
+                }
                 if(done) break;
                 try {
                     value.split("\\n\\n").forEach(json_str => {
                         if(json_str) {
                             const { choices } = JSON.parse(json_str);
-                            const content = choices[0].delta.content
+                            let content = choices[0].delta.content
+                            content = content.replaceAll("  ", "\\xa0\\xa0");
                             response += content;
-                            pending_conversation.textContent = response;
+                            if(content.includes("\\n")) {
+                                const content_parts = content.split("\\n")
+                                const arr_len = content_parts.length - 1;
+                                for(let i = 0; i < content_parts.length; i++) {
+                                    pending_conversation.append(content_parts[i])
+                                    if(i < arr_len) {
+                                        pending_conversation.append(document.createElement("br"));
+                                    }
+                                }
+                            } else pending_conversation.append(content);
                             conversation_main.scrollTo({
                                 behavior: "smooth", 
                                 top: conversation_main.scrollHeight