[REFACTOR][HOTFIX] 52 support python38 for llama index packs

.gitignore

@@ -3,4 +3,8 @@
 .DS_Store
 dist/
 README_hidden.md
-**/__pycache__/
+**/__pycache__/
+build/
+*.egg-info
+storage_graph/*
+chapter/*

.pre-commit-config.yaml

@@ -0,0 +1,111 @@
+---
+default_language_version:
+  python: python3
+
+exclude: "llama-index-integrations/vector_stores/llama-index-vector-stores-wordlift/manager_client/.*"
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-byte-order-marker
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-toml
+        exclude: llama-index-core/llama_index/core/_static
+      - id: check-yaml
+        exclude: llama-index-core/llama_index/core/_static
+      - id: detect-private-key
+      - id: end-of-file-fixer
+        exclude: llama-index-core/llama_index/core/_static
+      - id: mixed-line-ending
+        exclude: llama-index-core/llama_index/core/_static
+      - id: trailing-whitespace
+        exclude: llama-index-core/llama_index/core/_static
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.1.5
+
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+        exclude: ".*poetry.lock|.*_static"
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 23.10.1
+    hooks:
+      - id: black-jupyter
+        name: black-src
+        alias: black
+        exclude: "^docs|.*poetry.lock|.*_static"
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.0.1
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          [
+            "types-requests",
+            "types-Deprecated",
+            "types-redis",
+            "types-setuptools",
+            "types-PyYAML",
+            "types-protobuf==4.24.0.4",
+          ]
+        exclude: ^(docs/|llama-index-core/llama_index/core/_static)
+        args:
+          [
+            --namespace-packages,
+            --explicit-package-bases,
+            --disallow-untyped-defs,
+            --ignore-missing-imports,
+            --python-version=3.8,
+          ]
+        entry: bash -c "export MYPYPATH=llama_index"
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 23.10.1
+    hooks:
+      - id: black-jupyter
+        name: black-docs-py
+        alias: black
+        files: ^(docs/|examples/)
+        # Using PEP 8's line length in docs prevents excess left/right scrolling
+        args: [--line-length=79]
+  - repo: https://github.com/adamchainz/blacken-docs
+    rev: 1.16.0
+    hooks:
+      - id: blacken-docs
+        name: black-docs-text
+        alias: black
+        types_or: [rst, markdown, tex]
+        additional_dependencies: [black==23.10.1]
+        # Using PEP 8's line length in docs prevents excess left/right scrolling
+        args: [--line-length=79]
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.3
+    hooks:
+      - id: prettier
+        exclude: llama-index-core/llama_index/core/_static|poetry.lock|llama-index-legacy/llama_index/legacy/_static|docs/docs
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        additional_dependencies: [tomli]
+        exclude: |
+          poetry.lock|
+          (\/.*?\.[\w:]+)/pyproject.toml|
+          llama-index-core/llama_index/core/_static|
+          llama-index-legacy/llama_index/legacy/_static|
+          (\/.*?\.[\w:]+)/poetry.lock
+        args:
+          [
+            "--ignore-words-list",
+            "astroid,gallary,momento,narl,ot,rouge,nin,gere,asend",
+          ]
+  - repo: https://github.com/srstevenson/nb-clean
+    rev: 3.1.0
+    hooks:
+      - id: nb-clean
+        args: [--preserve-cell-outputs, --remove-empty-cells]
+  - repo: https://github.com/pappasam/toml-sort
+    rev: v0.23.1
+    hooks:
+      - id: toml-sort-fix
+        exclude: ".*poetry.lock|.*_static"

README.md

@@ -25,27 +25,34 @@ The above process includes the routing agent, knoweldge graph and memory module
 Raw source code for these components can also be found in their respective directories including benchmarks, notebooks, and updates.
 
 ## Installation
+
 1. With pip:
+
 ```
 pip install memary
 ```
 
 2. Locally:
-   1. Create your [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) and activate it. Note that Python versions 3.12 or greater are not supported by a key dependancy, llama-index and reccomended to run of python versions <= 3.11.9.
+
+   1. Create your [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) and activate it. Note that Python versions 3.12 or greater are not supported by a key dependency, llama-index and recommended to run of python versions <= 3.11.9.
 
    2. Install Python dependencies:
+
    ```
    pip install -r requirements.txt
    ```
 
 ## Demo
+
 **Notes:** memary currently assumes the local installation method and currently supports any models available through Ollama:
+
 - LLM running locally using Ollama (Llama 3 8B/40B as suggested defaults) **OR** `gpt-3.5-turbo`
 - Vision model running locally using Ollama (LLaVA as suggested default) **OR** `gpt-4-vision-preview`
 
 memary will default to the locally run models unless explicitly specified.
 
 **To run the Streamlit app:**
+
 1. [Optional] If running models locally using Ollama, follow this the instructions in this [repo](https://github.com/ollama/ollama).
 
 2. Ensure that a `.env` exists with any necessary API keys and Neo4j credentials.
@@ -76,20 +83,21 @@ Google Maps:
 
 Alpha Vantage: (this key is for getting real time stock data)
   https://www.alphavantage.co/support/#api-key
-  Reccomend use https://10minutemail.com/ to generate a temporary email to use
+  Recommend use https://10minutemail.com/ to generate a temporary email to use
 ```
 
-4.  Update user persona which can be found in `streamlit_app/data/user_persona.txt` using the user persona template which can be found in `streamlit_app/data/user_persona_template.txt`. Instructions have been provided - replace the curly brackets with relevant information. 
+4.  Update user persona which can be found in `streamlit_app/data/user_persona.txt` using the user persona template which can be found in `streamlit_app/data/user_persona_template.txt`. Instructions have been provided - replace the curly brackets with relevant information.
 
-5. . [Optional] Update system persona, if needed, which can be found in `streamlit_app/data/system_persona.txt`. 
-6. Run:
+5.  . [Optional] Update system persona, if needed, which can be found in `streamlit_app/data/system_persona.txt`.
+6.  Run:
 
 ```
 cd streamlit_app
 streamlit run app.py
 ```
 
 ## Usage
+
 ```python
 from memary.agent.chat_agent import ChatAgent
 
@@ -107,18 +115,24 @@ chat_agent = ChatAgent(
     past_chat_json,
 )
 ```
+
 Pass in subset of `['search', 'vision', 'locate', 'stocks']` as `include_from_defaults` for different set of default tools upon initialization.
+
 ### Adding Custom Tools
+
 ```python
 def multiply(a: int, b: int) -> int:
     """Multiply two integers and returns the result integer"""
     return a * b
 
+
 chat_agent.add_tool({"multiply": multiply})
 ```
-More information about creating custom tools for the LlamaIndex ReAct Agent  can be found [here](https://docs.llamaindex.ai/en/stable/examples/agent/react_agent/).
+
+More information about creating custom tools for the LlamaIndex ReAct Agent can be found [here](https://docs.llamaindex.ai/en/stable/examples/agent/react_agent/).
 
 ### Removing Tools
+
 ```python
 chat_agent.remove_tool("multiply")
 ```
@@ -191,7 +205,7 @@ The memory module comprises the Memory Stream and Entity Knowledge Store. The me
   - Personalize Responses: Use the key categorized entities and themes associated with the user to tailor agent responses more closely to the user's current interests and knowledge level/expertise.
   - Anticipate Needs: Leverage trends and shifts identified in the summaries to anticipate users' future questions or needs.
 - Future contributions
-  - We currently extract the top N entities from the entitiy knowledge store and pass these entities into the context window for inference. memary can future benefit from more advanced memory compression techniques such as passing only entities that are in the agent's response to the context window. We look forward to related community contributions.
+  - We currently extract the top N entities from the entity knowledge store and pass these entities into the context window for inference. memary can future benefit from more advanced memory compression techniques such as passing only entities that are in the agent's response to the context window. We look forward to related community contributions.
 
 ![Memory Compression](diagrams/memory_compression.png)
 
@@ -231,7 +245,7 @@ Currently memary is structured so that the ReAct agent can only process one quer
   - In a parallel system, the agent will be able to parse multiple queries at once. The query decomposer (QD) will pass all subqueries (or original query if no subqueries exist) to the agent at once.
   - Simultaneously, QD will pass the original query to the reranking module to rerank the agent responses based on their relevance to the pre-decomposed query.
 - Future contributions
-  - Once agent multiprocessing is integrated, QD will be valuable to leverage. All user queries will be passed to QD, and the (sub)queries wil be passed to the routing agent for parallel processing.
+  - Once agent multiprocessing is integrated, QD will be valuable to leverage. All user queries will be passed to QD, and the (sub)queries will be passed to the routing agent for parallel processing.
   - Self-Learning: Whenever queries are decomposed, those examples will be appended to the engine’s example store as a feedback loop for improved future performance.
 
 ### Reranking

dev/KG_memory_stream/data/system_persona.txt

@@ -7,4 +7,4 @@ Contexts: A history of interactions categorized by role, content, and date.
 Memory Stream: A list detailing entities and their interaction dates.
 Knowledge Entity Store: A record of entities, including their mention count and the date of the last mention.
 Interaction Keys: 'user' for user questions, 'rag' for responses from our knowledge graph, and 'system' for system-generated answers.
-Your responses should be informed, nuanced, and tailored, demonstrating a thorough understanding of the user's questions and the overarching conversation context. When addressing the user's latest inquiry, your answer must integrate the current conversation's context, historical interactions, and pertinent knowledge graph insights.
+Your responses should be informed, nuanced, and tailored, demonstrating a thorough understanding of the user's questions and the overarching conversation context. When addressing the user's latest inquiry, your answer must integrate the current conversation's context, historical interactions, and pertinent knowledge graph insights.

dev/KG_memory_stream/data/user_persona.txt

@@ -6,4 +6,4 @@ I really love sports such as badminton, tennis and workout and spend  4 or 5 day
 
 [Answer Instructions]
 I hope you can remember my questions and their answers so that I can leverage my personal past knowledge from you to be helpful in my life.
-I always prefer short and concise answers, not over two sentences.
+I always prefer short and concise answers, not over two sentences.

dev/KG_memory_stream/docs/update_021924.md

@@ -1,4 +1,5 @@
 ## Update
+
 - Implemented first version of Knowledge Graph RAG
   - LlamaIndex for graph framework
   - Nebula Graph for graph store
@@ -26,9 +27,8 @@
     - Extract chatper 1, 2 and 3 => N=11, E=9
       ![pic1](src/chapter1_2_3.png)
 
-
-
 ## Todo
+
 - [x] Modify [Nebula_graph_store_tutorial.ipynb](../Nebula_graph_store_tutorial.ipynb) to be easily produce Knowledge graphs
 - [ ] Implement the memory stream structures on ChatMemoryBuffer
-- [x] Test Neo4j of compatible with Knowledge Graph Retriever
+- [x] Test Neo4j of compatible with Knowledge Graph Retriever

dev/KG_memory_stream/docs/update_022024.md

@@ -1,13 +1,15 @@
 ## Update
+
 - Fix the stable version of llama-index
   - Specified all the requirements in [requirements.txt](../requirements.txt)
 - Test the memory feature of llama knowledge graph
   ![alt graph chat bot](src/GraphRAG.png)
   But sadly, we can't see the sub-graph after finishing asking questions. This sub-graph is what we wanted. I am finding the reason.
 
 ## Todo
+
 - [x] Modify [Nebula_graph_store_tutorial.ipynb](../Nebula_graph_store_tutorial.ipynb) to be easily produce Knowledge graphs
   - We can use [graph_rag_chat.py](../graph_rag_chatbot.py) for production.
 - [ ] Implement the memory stream structures on ChatMemoryBuffer
 - [x] Test Neo4j of compatible with Knowledge Graph Retriever
-- [ ] Visualize sub-graph to utilize for memory stream (**new**)
+- [ ] Visualize sub-graph to utilize for memory stream (**new**)

dev/KG_memory_stream/docs/update_022524.md

@@ -15,7 +15,8 @@ index = KnowledgeGraphIndex.from_documents(
     max_triplets_per_chunk=2,
 )
 ```
-I found there is a `insert_nodes` method to inject the processed graph to pre-existed one.  
+
+I found there is a `insert_nodes` method to inject the processed graph to pre-existed one.
 We can create our own Knowledge managing class to inject knowledge graph or remove or update it.
 
 <img src="src/0225_chapter1and2.png" width=40%>
@@ -26,6 +27,7 @@ We can create our own Knowledge managing class to inject knowledge graph or remo
 ## Memory Stream Class
 
 [memory_stream.py](../src/memory/memory_stream.py)
+
 ```
 # Example usage
 # Create a MemoryStream object
@@ -57,5 +59,6 @@ python -m pytest tests/
 ```
 
 # TODO
+
 - [ ] Create MemoryEntityStorage class
-- [ ] Preprocessing of sub-graph's output to store them in MemoryStream
+- [ ] Preprocessing of sub-graph's output to store them in MemoryStream

dev/KG_memory_stream/docs/update_022924.md

@@ -1,6 +1,6 @@
 ## Combined MemoryStream with RAG
 
-I made a half version of MemoryStream with RAG.  
+I made a half version of MemoryStream with RAG.
 We should save the entities of 'response,' but I couldn't figure out how I can get those entities from Llamaindex.
 
 We are retrieving answers through these processes.
@@ -14,7 +14,7 @@ We are retrieving answers through these processes.
 
 Between 4 and 5, we should get the entities of response, but there is no function in llamaindex. Steps 4 and 5 are in one function, `query` of `RetrieverQueryEngine`.
 
-But we can get the entities of query results of our Knowledge Graph easily. For the agile method, I just implement the memory stream using Knowledge Graph's entities.  
+But we can get the entities of query results of our Knowledge Graph easily. For the agile method, I just implement the memory stream using Knowledge Graph's entities.
 For example, if we ask, "Tell me about Harry", then we do a query to neo4j and can get all the relationships from our Knowledge Graph. And I am using this.
 
 ## Demo Result

dev/KG_memory_stream/openrc

@@ -4,4 +4,4 @@ export GRAPHD_HOST="127.0.0.1"
 export GRAPHD_PORT="9669"
 export NEBULA_USER="root"
 export NEBULA_PASSWORD="nebula"
-export NEBULA_ADDRESS=$GRAPHD_HOST:$GRAPHD_PORT
+export NEBULA_ADDRESS=$GRAPHD_HOST:$GRAPHD_PORT

dev/KG_memory_stream/tests/memory/test_entity_knowledge_store.py

@@ -6,11 +6,9 @@
 
 
 class TestEntityKnowledgeStore(unittest.TestCase):
-
     def setUp(self):
         self.file_name = "tests/memory/test_knowledge_memory.json"
-        self.entity_knowledge_store = EntityKnowledgeStore(
-            file_name=self.file_name)
+        self.entity_knowledge_store = EntityKnowledgeStore(file_name=self.file_name)
 
     def tearDown(self):
         # Clean up test file after each test
@@ -20,29 +18,26 @@ def tearDown(self):
             pass
 
     def test_add_memory(self):
-        data = [
-            MemoryItem("test_entity",
-                       datetime.now().replace(microsecond=0))
-        ]
+        data = [MemoryItem("test_entity", datetime.now().replace(microsecond=0))]
         self.entity_knowledge_store.add_memory(data)
         assert len(self.entity_knowledge_store.knowledge_memory) == 1
-        assert isinstance(self.entity_knowledge_store.knowledge_memory[0],
-                          KnowledgeMemoryItem)
+        assert isinstance(
+            self.entity_knowledge_store.knowledge_memory[0], KnowledgeMemoryItem
+        )
 
     def test_convert_memory_to_knowledge_memory(self):
-        data = [
-            MemoryItem("test_entity",
-                       datetime.now().replace(microsecond=0))
-        ]
-        converted_data = self.entity_knowledge_store._convert_memory_to_knowledge_memory(
-            data)
+        data = [MemoryItem("test_entity", datetime.now().replace(microsecond=0))]
+        converted_data = (
+            self.entity_knowledge_store._convert_memory_to_knowledge_memory(data)
+        )
         assert len(converted_data) == 1
         assert isinstance(converted_data[0], KnowledgeMemoryItem)
 
     def test_update_knowledge_memory(self):
         data = [
-            KnowledgeMemoryItem("knowledge_entity", 1,
-                                datetime.now().replace(microsecond=0))
+            KnowledgeMemoryItem(
+                "knowledge_entity", 1, datetime.now().replace(microsecond=0)
+            )
         ]
         self.entity_knowledge_store._update_knowledge_memory(data)
         assert len(self.entity_knowledge_store.knowledge_memory) == 1

dev/KG_memory_stream/tests/memory/test_memory.json

@@ -1,6 +1,6 @@
 [
-    {
-        "entity": "test_entity",
-        "date": "2024-02-26T12:22:14"
-    }
-]
+  {
+    "entity": "test_entity",
+    "date": "2024-02-26T12:22:14"
+  }
+]