|
| 1 | +# Install Outlines: |
| 2 | +# pip install outlines |
| 3 | + |
| 4 | +# Download Model: |
| 5 | +# huggingface-cli download bartowski/Phi-3.1-mini-4k-instruct-exl2 --revision 6_5 --local-dir Phi-3.1-mini-4k-instruct-exl2-6_5 |
| 6 | + |
| 7 | +import sys, os |
| 8 | +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| 9 | + |
| 10 | + |
| 11 | + |
| 12 | +from exllamav2 import ExLlamaV2, ExLlamaV2Config, ExLlamaV2Cache, ExLlamaV2Tokenizer |
| 13 | +from exllamav2.generator import ExLlamaV2DynamicGenerator, ExLlamaV2Sampler |
| 14 | + |
| 15 | +from outlines.processors import JSONLogitsProcessor |
| 16 | +from outlines.models.exllamav2 import patch_tokenizer as patch_exl2_tokenizer_for_outlines |
| 17 | + |
| 18 | +from pydantic import BaseModel, Field, RootModel |
| 19 | +from typing import Optional, Union, Literal |
| 20 | +from datetime import time |
| 21 | + |
| 22 | + |
| 23 | +################################################ |
| 24 | +# Create Structured JSON Generator With Outlines |
| 25 | +################################################ |
| 26 | + |
| 27 | +# Additional Examples: https://outlines-dev.github.io/outlines/cookbook/ |
| 28 | +# JSON Generation Docs: https://outlines-dev.github.io/outlines/reference/json/ |
| 29 | +# `outlines.processors` also supports guaranteed regex patterns and lark grammars |
| 30 | + |
| 31 | +# Example: Home Assistant extension for natural language commands -> actions |
| 32 | +class LightAction(BaseModel): |
| 33 | + entity: Literal["light"] = "light" |
| 34 | + action: Literal["turn_on", "turn_off", "set_brightness"] |
| 35 | + brightness: Optional[int] = Field(None, ge=0, le=100) |
| 36 | + execute_at: Optional[time] = None |
| 37 | + |
| 38 | + |
| 39 | +class OvenAction(BaseModel): |
| 40 | + entity: Literal["oven"] = "oven" |
| 41 | + action: Literal["turn_on", "turn_off", "set_temperature"] |
| 42 | + temperature: Optional[float] = Field(None, ge=50, le=300) |
| 43 | + execute_at: Optional[time] = None |
| 44 | + |
| 45 | + |
| 46 | +class HomeAssistantAction(BaseModel): |
| 47 | + instruction: Union[LightAction, OvenAction] |
| 48 | + |
| 49 | + |
| 50 | +def create_generator(model_dir="/mnt/str/models/mistral-7b-exl2/4.0bpw"): |
| 51 | + config = ExLlamaV2Config(model_dir) |
| 52 | + config.arch_compat_overrides() |
| 53 | + model = ExLlamaV2(config) |
| 54 | + cache = ExLlamaV2Cache(model, max_seq_len=32768, lazy=True) |
| 55 | + model.load_autosplit(cache, progress=True) |
| 56 | + |
| 57 | + print("Loading tokenizer...") |
| 58 | + tokenizer = ExLlamaV2Tokenizer(config) |
| 59 | + tokenizer.vocabulary = tokenizer.extended_piece_to_id |
| 60 | + |
| 61 | + # Initialize the generator with all default parameters |
| 62 | + return ExLlamaV2DynamicGenerator( |
| 63 | + model=model, |
| 64 | + cache=cache, |
| 65 | + tokenizer=tokenizer, |
| 66 | + ) |
| 67 | + |
| 68 | + |
| 69 | +generator = create_generator("./Phi-3.1-mini-4k-instruct-exl2-6_5") |
| 70 | + |
| 71 | +gen_settings = ExLlamaV2Sampler.Settings() |
| 72 | +gen_settings.logits_processor = JSONLogitsProcessor( |
| 73 | + HomeAssistantAction, |
| 74 | + patch_exl2_tokenizer_for_outlines(generator.tokenizer) |
| 75 | +) |
| 76 | + |
| 77 | + |
| 78 | +rules = "JSON for an instruction with an entity (light or oven) and action (turn_on, turn_off, set_brightness, set temperature). *Optionally* you may set an execute_at time-of-day if the user specifies, otherwise set to null" |
| 79 | +prompts = [ |
| 80 | + f"<|user|> {rules} Turn the lights lower please!<|end|><|assistant|>", |
| 81 | + f"<|user|> {rules} I need the oven set for homemade pizza when I get home from work at 6PM.<|end|><|assistant|>", |
| 82 | + f"<|user|> {rules} Oh no the lights are off and I can't find the switch!<|end|><|assistant|>", |
| 83 | +] |
| 84 | + |
| 85 | +outputs = generator.generate( |
| 86 | + prompt=prompts, |
| 87 | + gen_settings=gen_settings, |
| 88 | + max_new_tokens=2048, |
| 89 | + completion_only=True, |
| 90 | + encode_special_tokens=False, |
| 91 | + stop_conditions=[generator.tokenizer.eos_token_id], |
| 92 | +) |
| 93 | + |
| 94 | +# raw json format |
| 95 | +for idx, output in enumerate(outputs): |
| 96 | + print(output) |
| 97 | +# Output: |
| 98 | +# {"instruction": {"entity": "light", "action": "set_brightness", "execute_at": null}} |
| 99 | +# {"instruction": {"entity": "oven", "action": "set_temperature", "execute_at": "18:00:00"} } |
| 100 | +# {"instruction": {"entity": "light", "action": "turn_on"}} |
| 101 | + |
| 102 | +# pydantic model format |
| 103 | +for idx, output in enumerate(outputs): |
| 104 | + print(repr(HomeAssistantAction.parse_raw(output))) |
| 105 | +# Output: |
| 106 | +# HomeAssistantAction(instruction=LightAction(entity='light', action='set_brightness', brightness=None, execute_at=None)) |
| 107 | +# HomeAssistantAction(instruction=OvenAction(entity='oven', action='set_temperature', temperature=None, execute_at=datetime.time(18, 0))) |
| 108 | +# HomeAssistantAction(instruction=LightAction(entity='light', action='turn_on', brightness=None, execute_at=None)) |
0 commit comments