Ngrok Link

[Ressnn]

https://21f5-34-87-170-212.ngrok-free.app

[Ressnn]

from fastapi import FastAPI
from transformers import AutoTokenizer, AutoModelForCausalLM

app = FastAPI()

# -------------------------------------------------
#   3.1 Load your model and tokenizer at startup
# -------------------------------------------------
print("Loading model and tokenizer...")
tokenizer = AutoTokenizer.from_pretrained(
    "prdev/R1-Llama-8B-Function-Calling2",
    trust_remote_code=True
)
model = AutoModelForCausalLM.from_pretrained(
    "prdev/R1-Llama-8B-Function-Calling2",
    trust_remote_code=True,
    device_map="auto",
    load_in_8bit=True,              # This instructs HF Transformers to load weights in 8-bit
    torch_dtype=torch.float16       # Computations will still be in FP16 where needed
)


tokenizer2 = AutoTokenizer.from_pretrained(
    "deepseek-ai/DeepSeek-R1-Distill-Llama-8B",
    trust_remote_code=True
)
model2 = AutoModelForCausalLM.from_pretrained(
    "deepseek-ai/DeepSeek-R1-Distill-Llama-8B",
    trust_remote_code=True,
    device_map="auto",
    load_in_8bit=True,
    torch_dtype=torch.float16
)

print("Model loaded successfully.")

# -------------------------------------------------
#   3.2 Define Pydantic models (see above)
# -------------------------------------------------
from typing import List, Dict
from pydantic import BaseModel

class ChatInteractionMessage(BaseModel):
    role: str
    content: str

class ChatInteractionRequest(BaseModel):
    chat_interaction: List[ChatInteractionMessage]

# -------------------------------------------------
#   3.3 The existing pipeline function
# -------------------------------------------------
def run_search_inout_pipe(
    chat_interaction: List[Dict[str, str]], 
    tokenizer, 
    model
) -> str:
    prompt = tokenizer.apply_chat_template(chat_interaction, tokenize=False, add_generation_prompt=True)
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
    outputs = model.generate(
        **inputs, 
        max_new_tokens=2048, 
        pad_token_id=tokenizer.eos_token_id
    )
    prompt_len = inputs['input_ids'].shape[-1]
    generated_ids = outputs[:, prompt_len:]
    return tokenizer.decode(generated_ids[0], skip_special_tokens=True)


# -------------------------------------------------
#   3.4 The /chat endpoint
# -------------------------------------------------
@app.post("/search_loop")
def chat_interaction_endpoint(request: ChatInteractionRequest):
    """
    Accepts a JSON array of messages in `request.chat_interaction`,
    generates the assistant's reply, appends it to the conversation,
    and returns the updated conversation as JSON.
    """
    # Extract the list of messages
    conversation = request.chat_interaction
    
    # 1) Generate the assistant's response
    response_text = run_search_inout_pipe(
        chat_interaction=conversation,
        tokenizer=tokenizer,
        model=model
    )
    
    # 2) Append the new response to the conversation
    conversation.append(
        ChatInteractionMessage(role="assistant", content=response_text)
    )
    
    # 3) Return the entire updated conversation
    return {"chat_interaction": conversation}


@app.post("/finish_search")
def chat_interaction_endpoint(request: ChatInteractionRequest):
    """
    Accepts a JSON array of messages in `request.chat_interaction`,
    generates the assistant's reply, appends it to the conversation,
    and returns the updated conversation as JSON.
    """
    # Extract the list of messages
    conversation = request.chat_interaction
    
    # 1) Generate the assistant's response
    response_text = run_search_inout_pipe(
        chat_interaction=conversation,
        tokenizer=tokenizer2,
        model=model2
    )
    
    # 2) Append the new response to the conversation
    conversation.append(
        ChatInteractionMessage(role="assistant", content=response_text)
    )
    
    # 3) Return the entire updated conversation
    return {"chat_interaction": conversation}

# -------------------------------------------------
#   3.5 Healthcheck Endpoint
# -------------------------------------------------
@app.get("/healthcheck")
def healthcheck():
    return {"status": "ok"}```

[Ressnn]

Expected Ping Cost

Running search inference...
Inference result:
Okay, the user mentioned revenue increased from 22M to 55M. Let me figure out how to plot this data. I'll use the search function to find the relevant data points. Once I get the data, I'll format it for plotting.

{"name": "search", "arguments": {"query": "revenue data for Q1"} }