From af5fb62e4c0b62fdd1e7ac686008256f8e88b832 Mon Sep 17 00:00:00 2001 From: Thomas Hanke Date: Thu, 10 Jul 2025 14:57:37 +0200 Subject: [PATCH] fixed ckan_agent result processing --- ckanext/chat/bot/agent.py | 14 +++++--------- ckanext/chat/bot/utils.py | 28 +++++++++++++++++----------- ckanext/chat/plugin.py | 1 + 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/ckanext/chat/bot/agent.py b/ckanext/chat/bot/agent.py index d61c769..3206bb5 100644 --- a/ckanext/chat/bot/agent.py +++ b/ckanext/chat/bot/agent.py @@ -295,6 +295,7 @@ class CKANResult(BaseModel): " - Present updates and changes, requesting user confirmation before proceeding, when running actions that chnage the data.\n" " - Request confirmation if SSL verification is disabled (`ssl_verify=False` for downloads).\n" "Guidelines:\n" + "- if `ckan_run` fails adopt your call by the suggestions made in the response, add default parameters as necessarry.\n" "- use 'get_ckan_actions' to find a dict with keys of action names and values the functions signature.\n" "- Use `ckan_run` with command `package_search` and parameters `{q:search_str, include_private: true}` for comprehensive dataset searches. If the user does not specify what he searches for use search_str="".\n" "- If u have no idea on what to do, ask a question on a suitable action to `ckan_run`" @@ -353,12 +354,6 @@ class CKANResult(BaseModel): # model_settings=OpenAIModelSettings(openai_reasoning_effort= "low") ) - -class SliceModel(BaseModel): - start: int - end: int - - doc_agent = Agent( model=model, deps_type=TextResource, @@ -506,8 +501,9 @@ def run_action(ctx: RunContext[Deps], action_name: str, parameters: Dict) -> Any response = toolkit.get_action(action_name)(context, parameters) except Exception as e: return {"error": str(e)} - clean_response = unpack_lazy_json(response) - clean_response = process_entity(clean_response) + unpacked_response = unpack_lazy_json(response) + log.debug(type(unpacked_response)) + clean_response = process_entity(unpacked_response) log.debug(clean_response) #log.debug("{} -> {}".format(len(str(response)), len(str(clean_response)))) return clean_response @@ -820,7 +816,7 @@ async def literature_analyse(doc: TextResource, question: str, ssl_verify=True) deps=doc, usage_limits=UsageLimits(request_limit=50), ), - timeout=30 + timeout=60 ) except asyncio.TimeoutError: msg="Timeout on literature_analyse attempt, retrying..." diff --git a/ckanext/chat/bot/utils.py b/ckanext/chat/bot/utils.py index dc03aa8..268296d 100644 --- a/ckanext/chat/bot/utils.py +++ b/ckanext/chat/bot/utils.py @@ -243,20 +243,24 @@ def unpack_lazy_json(obj): return obj -def process_entity(data: Any, depth: int = 0, max_depth: int = 2) -> Any: +def process_entity(data: Any, depth: int = 0, max_depth: int = 4) -> Any: + log.debug(f"{type(data)},{depth},{max_depth}") if depth > max_depth: log.warning("Max recursion depth reached") - return data + #data=truncate_by_depth(data,max_depth) + return None if isinstance(data, dict): data = unpack_lazy_json(data) + #log.debug(data.keys()) if "resources" in data: try: + #log.debug("Dataset") dataset_dict = DynamicDataset(**data).model_dump( exclude_unset=True, exclude_defaults=False, exclude_none=True ) dataset_dict = {k: v for k, v in dataset_dict.items() if bool(v)} - return process_entity(dataset_dict, depth + 1, max_depth) + return truncate_by_depth(dataset_dict,max_depth-depth) except ValidationError as validation_error: log.warning( f"Validation error converting to DynamicDataset: {validation_error.json()}" @@ -265,24 +269,26 @@ def process_entity(data: Any, depth: int = 0, max_depth: int = 2) -> Any: log.warning(f"Conversion to DynamicDataset failed: {ex}") elif "package_id" in data or "url" in data: try: + #log.debug("Resource") resource_dict = DynamicResource(**data).model_dump( exclude_unset=True, exclude_defaults=False, exclude_none=True ) resource_dict = {k: v for k, v in resource_dict.items() if bool(v)} - return process_entity(resource_dict, depth + 1, max_depth) + return truncate_by_depth(resource_dict,max_depth-depth) except ValidationError as validation_error: log.warning( f"Validation error converting to DynamicResource: {validation_error.json()}" ) except Exception as ex: log.warning(f"Conversion to DynamicResource failed: {ex}") - - new_dict = {} - for key, value in data.items(): - processed_value = process_entity(value, depth + 1, max_depth) - if processed_value not in ([], {}, "", None): - new_dict[key] = processed_value - return new_dict + else: + #log.debug("Dictionary") + new_dict = {} + for key, value in data.items(): + processed_value = process_entity(value, depth + 1, max_depth) + if processed_value not in ([], {}, "", None): + new_dict[key] = processed_value + return new_dict elif isinstance(data, list): new_list = [] diff --git a/ckanext/chat/plugin.py b/ckanext/chat/plugin.py index 84074d5..c459f69 100644 --- a/ckanext/chat/plugin.py +++ b/ckanext/chat/plugin.py @@ -34,6 +34,7 @@ def declare_config_options(self, declaration: Declaration, key: Key): declaration.declare(group.api_token, "your-api-token") declaration.declare(group.embedding_model, "text-embedding-3-small") declaration.declare(group.embedding_api, "") + declaration.declare(group.milvus_url, "") declaration.declare(group.collection_name, "") # IBlueprint