chore: function calling cleanup

Author: sangjanai

docs/docs/guides/function-calling.md

@@ -63,8 +63,14 @@ tools = [
 
 completion_payload = {
     "messages": [
-        {"role": "system", "content": "You are a helpful customer support assistant. Use the supplied tools to assist the user."},
-        {"role": "user", "content": "Hi, can you tell me the delivery date for my order?"},
+        {
+            "role": "system",
+            "content": 'You have access to the following CUSTOM functions:\n\n<CUSTOM_FUNCTIONS>\n\nIf a you choose to call a function ONLY reply in the following format:\n<{start_tag}={function_name}>{parameters}{end_tag}\nwhere\n\nstart_tag => `<function`\nparameters => a JSON dict with the function argument name as key and function argument value as value.\nend_tag => `</function>`\n\nHere is an example,\n<function=example_function_name>{"example_name": "example_value"}</function>\n\nReminder:\n- Function calls MUST follow the specified format\n- Required parameters MUST be specified\n- You can call one or more functions at a time, but remember only chose correct function\n- Put the entire function call reply on one line\n- Always add your sources when using search results to answer the user query\n- If you can not find correct parameters or arguments corresponding to function in the user\'s message, ask user again to provide, do not make assumptions.\n- No explanation are needed when calling a function.\n\nYou are a helpful assistant.',
+        },
+        {
+            "role": "user", 
+            "content": "Hi, can you tell me the delivery date for my order?"
+        },
     ]
 }
 
@@ -126,10 +132,22 @@ Once the user provides their order ID:
 ```python
 completion_payload = {
     "messages": [
-        {"role": "system", "content": "You are a helpful customer support assistant. Use the supplied tools to assist the user."},
-        {"role": "user", "content": "Hi, can you tell me the delivery date for my order?"},
-        {"role": "assistant", "content": "Of course! Please provide your order ID so I can look it up."},
-        {"role": "user", "content": "i think it is order_70705"},
+        {
+            "role": "system",
+            "content": 'You have access to the following CUSTOM functions:\n\n<CUSTOM_FUNCTIONS>\n\nIf a you choose to call a function ONLY reply in the following format:\n<{start_tag}={function_name}>{parameters}{end_tag}\nwhere\n\nstart_tag => `<function`\nparameters => a JSON dict with the function argument name as key and function argument value as value.\nend_tag => `</function>`\n\nHere is an example,\n<function=example_function_name>{"example_name": "example_value"}</function>\n\nReminder:\n- Function calls MUST follow the specified format\n- Required parameters MUST be specified\n- You can call one or more functions at a time, but remember only chose correct function\n- Put the entire function call reply on one line\n- Always add your sources when using search results to answer the user query\n- If you can not find correct parameters or arguments corresponding to function in the user\'s message, ask user again to provide, do not make assumptions.\n- No explanation are needed when calling a function.\n\nYou are a helpful assistant.',
+        },
+        {
+            "role": "user", 
+            "content": "Hi, can you tell me the delivery date for my order?"
+        },
+        {
+            "role": "assistant", 
+            "content": "Of course! Please provide your order ID so I can look it up."
+        },
+        {
+            "role": "user", 
+            "content": "i think it is order_70705"
+        },
     ]
 }
 

engine/controllers/server.cc

@@ -179,7 +179,6 @@ void server::ProcessStreamRes(std::function<void(const HttpResponsePtr&)> cb,
 void server::ProcessNonStreamRes(std::function<void(const HttpResponsePtr&)> cb,
                                  SyncQueue& q) {
   auto [status, res] = q.wait_and_pop();
-  function_calling_utils::PostProcessResponse(res);
   LOG_DEBUG << "response: " << res.toStyledString();
   auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
   resp->setStatusCode(

engine/extensions/local-engine/local_engine.cc

@@ -531,6 +531,7 @@ void LocalEngine::LoadModel(std::shared_ptr<Json::Value> json_body,
 
   params.push_back("--pooling");
   params.push_back("mean");
+  params.push_back("--jinja");
 
   std::vector<std::string> v;
   v.reserve(params.size() + 1);

engine/services/inference_service.cc

@@ -13,8 +13,6 @@ cpp::result<void, InferResult> InferenceService::HandleChatCompletion(
     engine_type = (*(json_body)).get("engine", kLlamaRepo).asString();
   }
   CTL_DBG("engine_type: " << engine_type);
-  function_calling_utils::PreprocessRequest(json_body);
-  CTL_DBG("engine_type: " << engine_type);
   auto tool_choice = json_body->get("tool_choice", Json::Value::null);
   auto model_id = json_body->get("model", "").asString();
   if (saved_models_.find(model_id) != saved_models_.end()) {

engine/test/components/test_function_calling.cc

@@ -27,13 +27,13 @@ inline void PrintMenu(
 
 inline std::optional<int> GetNumericValue(const std::string& sval) {
   try {
-      return std::stoi(sval);
+    return std::stoi(sval);
   } catch (const std::invalid_argument&) {
-      // Not a valid number
-      return std::nullopt; 
+    // Not a valid number
+    return std::nullopt;
   } catch (const std::out_of_range&) {
-      // Number out of range
-      return std::nullopt;
+    // Number out of range
+    return std::nullopt;
   }
 }
 
@@ -73,14 +73,16 @@ inline std::optional<std::string> PrintModelSelection(
   }
 
   // Validate if the selection consists solely of numeric characters
-  if(!std::all_of(selection.begin(), selection.end(), ::isdigit)){
+  if (!std::all_of(selection.begin(), selection.end(), ::isdigit)) {
     return std::nullopt;
   }
 
   // deal with out of range numeric values
   std::optional<int> numeric_value = GetNumericValue(selection);
-  
-  if (!numeric_value.has_value() || (unsigned) numeric_value.value() > availables.size() || numeric_value.value() < 1) {
+
+  if (!numeric_value.has_value() ||
+      (unsigned)numeric_value.value() > availables.size() ||
+      numeric_value.value() < 1) {
     return std::nullopt;
   }
 
@@ -101,13 +103,15 @@ inline std::optional<std::string> PrintSelection(
   }
 
   // Validate if the selection consists solely of numeric characters
-  if(!std::all_of(selection.begin(), selection.end(), ::isdigit)){
+  if (!std::all_of(selection.begin(), selection.end(), ::isdigit)) {
     return std::nullopt;
   }
-  
+
   // deal with out of range numeric values
   std::optional<int> numeric_value = GetNumericValue(selection);
-  if (!numeric_value.has_value() ||(unsigned) numeric_value.value() > options.size() || numeric_value.value() < 1) {
+  if (!numeric_value.has_value() ||
+      (unsigned)numeric_value.value() > options.size() ||
+      numeric_value.value() < 1) {
     return std::nullopt;
   }