From 368d26a15ffd8c8f53a5813f16da7ffcd88aead4 Mon Sep 17 00:00:00 2001 From: cbh778899 Date: Tue, 27 Aug 2024 15:51:17 +1000 Subject: [PATCH 1/7] retrieve max_tokens from request Signed-off-by: cbh778899 --- routes/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routes/index.js b/routes/index.js index 04e64e5..58f5476 100644 --- a/routes/index.js +++ b/routes/index.js @@ -36,8 +36,9 @@ function indexRoute() { if(isRouteEnabled("index", "chatbox")) { router.get('/chatbox', (req, res)=>{ + const { base_url, max_tokens } = req.query; res.setHeader("Content-Type", "application/json; charset=utf-8") - res.send(generateScript(req.query.base_url)); + res.send(generateScript(base_url, max_tokens)); }) } From f3de3a268e8193cd990f3c93af706b407f658a76 Mon Sep 17 00:00:00 2001 From: cbh778899 Date: Tue, 27 Aug 2024 15:51:42 +1000 Subject: [PATCH 2/7] add max_tokens & new lines when got \n Signed-off-by: cbh778899 --- tools/web_embed.js | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tools/web_embed.js b/tools/web_embed.js index c84cc6d..fdc46f8 100644 --- a/tools/web_embed.js +++ b/tools/web_embed.js @@ -2,7 +2,7 @@ * Generates the script with base_url, to simply embed a chatbot in web page * @param {String} base_url The url to send request, default `http://localhost:8000` */ -export function generateScript(base_url = 'http://localhost:8000') { +export function generateScript(base_url = 'http://localhost:8000', max_tokens = 128) { return ` (function() { 'use strict'; @@ -291,7 +291,9 @@ const styles = \` function createElement(tagName, className, textContent) { const elem = document.createElement(tagName); elem.className = className; - elem.textContent = textContent; + textContent.split("\\n").forEach(content=>{ + elem.append(content, document.createElement("br")) + }) return elem; } @@ -313,7 +315,8 @@ const styles = \` {role: 'system', content: "You are a helpful assistant solves problem"}, {role: 'user', content: message} ], - stream: true + stream: true, + max_tokens: ${max_tokens} }) }); if(resp.ok) { @@ -328,7 +331,16 @@ const styles = \` const { choices } = JSON.parse(json_str); const content = choices[0].delta.content response += content; - pending_conversation.textContent = response; + if(content.includes("\\n")) { + const content_parts = content.split("\\n") + const arr_len = content_parts.length - 1; + for(let i = 0; i < content_parts.length; i++) { + pending_conversation.append(content_parts[i]) + if(i < arr_len) { + pending_conversation.append(document.createElement("br")); + } + } + } else pending_conversation.append(content); conversation_main.scrollTo({ behavior: "smooth", top: conversation_main.scrollHeight From d1a1968393081449d20ef9588a9e747774e6d9ac Mon Sep 17 00:00:00 2001 From: cbh778899 Date: Tue, 27 Aug 2024 16:21:27 +1000 Subject: [PATCH 3/7] remove dot animation on received message Signed-off-by: cbh778899 --- tools/web_embed.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/web_embed.js b/tools/web_embed.js index fdc46f8..f4adb62 100644 --- a/tools/web_embed.js +++ b/tools/web_embed.js @@ -105,6 +105,7 @@ const styles = \` border-radius: 20px; border: 1px solid gray; padding: 0px 40px 0px 10px; + box-sizing: border-box; } @@ -176,6 +177,7 @@ const styles = \` top: 40px; display: block; overflow-y: auto; + overflow-x: hidden; padding: 20px; box-sizing: border-box; } @@ -188,6 +190,12 @@ const styles = \` text-align: center; text-decoration: none; display: block; + transition-duration: .3s; +} + +.chatbox > .expanded-page > .conversations > .power-by:hover { + transform: scale(1.2); + font-weight: bold; } .chatbox > .expanded-page > .conversations > .bubble { @@ -199,6 +207,8 @@ const styles = \` margin: auto; font-size: 17px; margin-bottom: 10px; + word-wrap: break-word; + max-width: calc(100% - 40px); } .chatbox > .expanded-page > .conversations > .bubble.empty { @@ -321,9 +331,13 @@ const styles = \` }); if(resp.ok) { const reader = resp.body.pipeThrough(new TextDecoderStream()).getReader(); - let response = '' + let response = '', started = false; while(true) { const { value, done } = await reader.read(); + if(!started) { + started = true; + pending_conversation.textContent = ''; + } if(done) break; try { value.split("\\n\\n").forEach(json_str => { From eb991107d2261ff7fcada6d04b089656d027043d Mon Sep 17 00:00:00 2001 From: cbh778899 Date: Tue, 27 Aug 2024 17:16:19 +1000 Subject: [PATCH 4/7] try-catch json parse errors to prevent when content is too long to receive Signed-off-by: cbh778899 --- actions/inference.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/actions/inference.js b/actions/inference.js index abc40e1..de9df44 100644 --- a/actions/inference.js +++ b/actions/inference.js @@ -83,7 +83,12 @@ async function doInference(req_body, callback, isStream) { const { value, done } = await reader.read(); if(done) break; const data = value.split("data: ").pop() - callback(JSON.parse(data)); + try { + callback(JSON.parse(data)); + } catch(error) { + console.log(error) + callback({content: "", stop: true}) + } } } else { const eng_resp = await post('completion', { body: req_body }); From 5a2041064d2c29ae73765ad3d785941d3e44b327 Mon Sep 17 00:00:00 2001 From: cbh778899 Date: Tue, 27 Aug 2024 17:18:02 +1000 Subject: [PATCH 5/7] replace all double-space to \xa0\xa0 Signed-off-by: cbh778899 --- tools/web_embed.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/web_embed.js b/tools/web_embed.js index f4adb62..79fd78a 100644 --- a/tools/web_embed.js +++ b/tools/web_embed.js @@ -343,7 +343,8 @@ const styles = \` value.split("\\n\\n").forEach(json_str => { if(json_str) { const { choices } = JSON.parse(json_str); - const content = choices[0].delta.content + let content = choices[0].delta.content + content = content.replaceAll(" ", "\\xa0\\xa0"); response += content; if(content.includes("\\n")) { const content_parts = content.split("\\n") From 2b4bcfcfdb286c01431a6845386822a5ebd07191 Mon Sep 17 00:00:00 2001 From: Zuhayer C <80013387+ZooHigher26@users.noreply.github.com> Date: Mon, 26 Aug 2024 09:54:48 +1000 Subject: [PATCH 6/7] Add demo_ui.py to project (#68) * Add demo_ui.py to project Uploaded demo_ui.py file to project example folder. Signed-off-by: Zuhayer C <80013387+ZooHigher26@users.noreply.github.com> * Replace "Customary xxxxxx: chatbot" with project name Signed-off-by: Zuhayer C <80013387+ZooHigher26@users.noreply.github.com> --------- Signed-off-by: Zuhayer C <80013387+ZooHigher26@users.noreply.github.com> --- example/demo_ui.py | 153 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 example/demo_ui.py diff --git a/example/demo_ui.py b/example/demo_ui.py new file mode 100644 index 0000000..9bc17db --- /dev/null +++ b/example/demo_ui.py @@ -0,0 +1,153 @@ +import streamlit as st +import random +import string +import datetime +import tiktoken +from openai import OpenAI + +RMIT="RMIT" + +encoding = tiktoken.get_encoding("r50k_base") + +client = OpenAI(base_url="http://localhost:8080/v1", api_key="not-needed", organization='SelectedModel') + +def writehistory(filename,text): + with open(filename, 'a', encoding='utf-8') as f: + f.write(text) + f.write('\n') + f.close() + +#AVATARS πŸ‘·πŸ¦ πŸ₯ΆπŸŒ€ +av_us = 'πŸ¦–' #"πŸ¦–" #A single emoji, e.g. "πŸ§‘β€πŸ’»", "πŸ€–", "πŸ¦–". +av_ass = 'πŸ‘·' + +# Set the webpage title +st.set_page_config( + page_title=f"Your LocalGPT with 🌟 {RMIT}", + page_icon="🌟", + layout="wide") + +# Create a header element +mytitle = '# Local AI Solutions and LLM-based Chatbot development 🌟' +st.markdown(mytitle, unsafe_allow_html=True) +st.markdown('### Conversation tokens context window') +# function to generate random alphanumeric sequence for the filename +def genRANstring(n): + """ + n = int number of char to randomize + """ + N = n + res = ''.join(random.choices(string.ascii_uppercase + + string.digits, k=N)) + return res + +# create THE SESSIoN STATES +if "logfilename" not in st.session_state: +## Logger file + logfile = f'{genRANstring(5)}_log.txt' + st.session_state.logfilename = logfile + #Write in the history the first 2 sessions + writehistory(st.session_state.logfilename,f'{str(datetime.datetime.now())}\n\nYour own LocalGPT with πŸŒ€ {RMIT}\n---\n🧠🫑: You are a helpful assistant.') + writehistory(st.session_state.logfilename,f'πŸŒ€: How may I help you today?') + +if "repeat" not in st.session_state: + st.session_state.repeat = 1.35 + +if "temperature" not in st.session_state: + st.session_state.temperature = 0.1 + +if "maxlength" not in st.session_state: + st.session_state.maxlength = 500 + + +# CREATE THE SIDEBAR +with st.sidebar: + st.image('img/RMIT_POS3.png', use_column_width=True) + st.session_state.temperature = st.slider('Temperature:', min_value=0.0, max_value=1.0, value=0.1, step=0.02) + st.session_state.maxlength = st.slider('Length reply:', min_value=150, max_value=1000, + value=500, step=50) + st.session_state.repeat = st.slider('Repeat Penalty:', min_value=0.0, max_value=2.0, value=1.35, step=0.01) + st.markdown(f"**Logfile**: {st.session_state.logfilename}") + btnClear = st.button("Clear History",type="primary", use_container_width=True) + + +# We store the conversation in the session state. +# This will be used to render the chat conversation. +# We initialize it with the first message we want to be greeted with. +#Note that the first 3 messages will never be used for the genration, they are only for the Chat interface +if "messages" not in st.session_state: + st.session_state.messages = [ + {"role": "system", "content": "You are assistant, a helpful assistant. You reply only to the user questions. You always reply in the language of the instructions.",}, + {"role": "user", "content": "Hi, I am P000252SE."}, + {"role": "assistant", "content": "Hi there, I am assistant, how may I help you today?"} + ] +# we define the function to clear from the screen the conversation history +def clearHistory(): + st.session_state.messages = [ + {"role": "system", "content": "You are assistant, a helpful assistant. You reply only to the user questions. You always reply in the language of the instructions.",}, + {"role": "user", "content": "Hi, I am P000252SE."}, + {"role": "assistant", "content": "Hi there, I am assistant, how may I help you today?"} + ] +if btnClear: + clearHistory() + + +# We loop through each message in the session state and render it as # a chat message. +for message in st.session_state.messages[1:]: + if message["role"] == "user": + with st.chat_message(message["role"],avatar=av_us): + st.markdown(message["content"]) + else: + with st.chat_message(message["role"],avatar=av_ass): + st.markdown(message["content"]) + +# We take questions/instructions from the chat input to pass to the LLM +if user_prompt := st.chat_input("Your message here. Shift+Enter to add a new line", key="user_input"): + + # Add our input to the session state + st.session_state.messages.append( + {"role": "user", "content": user_prompt} + ) + + # Add our input to the chat window + with st.chat_message("user", avatar=av_us): + st.markdown(user_prompt) + writehistory(st.session_state.logfilename,f'πŸ‘·: {user_prompt}') + + + with st.chat_message("assistant",avatar=av_ass): + message_placeholder = st.empty() + with st.spinner("Inferencing..."): + response = '' + conv_messages = [] + conv_messages.append(st.session_state.messages[-1]) + full_response = "" + + completion = client.chat.completions.create( + model="local-model", + messages=conv_messages, #st.session_state.messages if you want to keep previous messages, + temperature=st.session_state.temperature, + frequency_penalty = st.session_state.repeat, + stop=['<|im_end|>',''], + max_tokens=st.session_state.maxlength, + stream=True, + ) + for chunk in completion: + if chunk.choices[0].delta.content: + full_response += chunk.choices[0].delta.content + message_placeholder.markdown(full_response + "🌟") + + toregister = full_response + f""" +``` + +prompt tokens: {len(encoding.encode(st.session_state.messages[-1]['content']))} +generated tokens: {len(encoding.encode(full_response))} +```""" + message_placeholder.markdown(toregister) + writehistory(st.session_state.logfilename,f'🌟: {toregister}\n\n---\n\n') + + + # Add the response to the session state + st.session_state.messages.append( + {"role": "assistant", "content": toregister} + ) From c8c5bcc204d40e08488fe0a0de516959e982283d Mon Sep 17 00:00:00 2001 From: cbh778899 Date: Fri, 30 Aug 2024 16:09:38 +1000 Subject: [PATCH 7/7] update README Signed-off-by: cbh778899 --- README.md | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index c9d33ea..ceae378 100644 --- a/README.md +++ b/README.md @@ -60,30 +60,45 @@ make dev ``` **NOTE:** `make dev` Requires Node.js environment installed, or at least have `node_modules` specified in `package.json` installed on your server. Please see [Local Machine](#local-machine) section. -## Embed your chatbot into website -To embed a chatbot to your website, simply add +## Lint +To start lint your code, simply run +```shell +npm run lint +``` + +## APIs + +### Docs +Go to the url of your project, default [http://localhost:8000](http://localhost:8000) if you didn't disabled the `Docs` route, then you can see docs and try it on. +See [demo video](#setup-and-api-usage-demo-video). + +### Monitor +This project got monitor build with swagger-stats, when you got this project running, just go to `:/stats`. +For example, [http://localhost:8000/stats](http://localhost:8000/stats) + +### Chatbox +> When you set up the project and didn't disabled the `chatbox` API, you can get a quick-setup chatbot with some basic styles on your own website, which calls the `/v1/chat/completions` API for inference. + +To set it up, simply add ```html - + ``` into the bottom of your html body element. So easy! -Please change the `http://localhost:8000` and the base_url in request query to your real host to make sure it works. If you want to hide the real link, in your javascript code you can do ```js -const chatbox_script = await (await fetch("http://localhost:8000/chatbox?base_url=http%3A%2F%2Flocalhost%3A8000")).blob(); +const chatbox_script = await (await fetch("http://localhost:8000/chatbox")).blob(); const chatbox_url = URL.createObjectURL(chatbox_script); const script_elem = document.createElement('script'); script_elem.src = chatbox_url; document.body.append(script_elem); ``` -And remember to use `URL.revokeObjectURL(chatbox_url)` if you don't need it anymore. - -## Lint -To start lint your code, simply run -```shell -npm run lint -``` - -## Monitor -This project got monitor build with swagger-stats, when you got this project running, just go to `:/stats`. -For example, [http://localhost:8000/stats](http://localhost:8000/stats) +And remember to use `URL.revokeObjectURL(chatbox_url)` if you don't need it anymore. + +Extra parameters ([request query](https://en.wikipedia.org/wiki/Query_string)) you can add to it are: +* `base_url`: `String` + > Add this when in production, otherwise the requests won't send to correct route. + > Default `http://localhost:8000`. +* `max_tokens`: `Integer` + > Add this when you want to limit tokens can be generated, is useful in production. + > Default `128`