Skip to content

Community: Add sqldocstore support for ParentChildRetriever #30300

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 24 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
c08b3be
add aprep_output to be called inside ainvoke()
hslee16 Apr 22, 2024
77419ec
format
hslee16 Apr 22, 2024
08ec60c
Merge branch 'master' into add_aprep_outputs_to_chain_base
hslee16 Apr 22, 2024
7db9733
Merge branch 'master' into add_aprep_outputs_to_chain_base
hslee16 Apr 22, 2024
b277bc5
Merge pull request #1 from hslee16/add_aprep_outputs_to_chain_base
hslee16 Apr 23, 2024
894eee7
add private repo build
hslee16 Apr 23, 2024
05b0a10
Merge pull request #2 from hslee16/private_repo
hslee16 Apr 23, 2024
6d0adaf
Merge branch 'langchain-ai:master' into master
hslee16 Sep 14, 2024
91b0ea3
Merge branch 'langchain-ai:master' into master
hslee16 Mar 15, 2025
40979a0
resolve sqldocstore postgres compatibility issue
hslee16 Mar 15, 2025
3b5019d
ParentChildRetriever support for sqldocstore
hslee16 Mar 16, 2025
0077286
revert gitignore change
hslee16 Mar 16, 2025
a54ef5a
await async calls
hslee16 Mar 16, 2025
b4d5073
Update libs/community/tests/unit_tests/storage/test_sql.py
hslee16 Mar 17, 2025
7285def
addressing comments from PR
hslee16 Mar 17, 2025
5796882
lint fixes
hslee16 Mar 17, 2025
1085926
more linting fixes
hslee16 Mar 17, 2025
9615a5b
use Union
hslee16 Mar 18, 2025
3daa625
Merge branch 'master' into sqldocstore_postgres_compat
hslee16 Mar 18, 2025
88b0884
Merge branch 'master' into sqldocstore_postgres_compat
hslee16 Mar 18, 2025
affb083
Update libs/community/tests/unit_tests/storage/test_sql.py
hslee16 Mar 19, 2025
73e218d
Merge branch 'master' into sqldocstore_postgres_compat
hslee16 Mar 19, 2025
51ddb32
Merge branch 'master' into sqldocstore_postgres_compat
hslee16 Mar 19, 2025
ab643d2
Merge branch 'master' into sqldocstore_postgres_compat
hslee16 Mar 20, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 26 additions & 13 deletions libs/community/langchain_community/storage/sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
cast,
)

from langchain_core.documents.base import Document
from langchain_core.stores import BaseStore
from sqlalchemy import (
LargeBinary,
Expand Down Expand Up @@ -176,55 +177,67 @@ def drop(self) -> None:

async def amget(self, keys: Sequence[str]) -> List[Optional[bytes]]:
assert isinstance(self.engine, AsyncEngine)
result: Dict[str, bytes] = {}
async with self._make_async_session() as session:
stmt = select(LangchainKeyValueStores).filter(
and_(
LangchainKeyValueStores.key.in_(keys),
LangchainKeyValueStores.namespace == self.namespace,
)
)
for v in await session.scalars(stmt):
result[v.key] = v.value
return [result.get(key) for key in keys]
results = await session.execute(stmt)
return [result[0].value for result in results.all()]

def mget(self, keys: Sequence[str]) -> List[Optional[bytes]]:
result = {}

with self._make_sync_session() as session:
stmt = select(LangchainKeyValueStores).filter(
and_(
LangchainKeyValueStores.key.in_(keys),
LangchainKeyValueStores.namespace == self.namespace,
)
)
for v in session.scalars(stmt):
result[v.key] = v.value
return [result.get(key) for key in keys]
results = session.execute(stmt).all()
return [result[0].value for result in results]

async def amset(self, key_value_pairs: Sequence[Tuple[str, bytes]]) -> None:
async with self._make_async_session() as session:
await self._amdelete([key for key, _ in key_value_pairs], session)
session.add_all(
[
LangchainKeyValueStores(namespace=self.namespace, key=k, value=v)
LangchainKeyValueStores(
namespace=self.namespace,
key=k,
value=self._bytes_or_document(v),
)
for k, v in key_value_pairs
]
)
await session.commit()

def mset(self, key_value_pairs: Sequence[Tuple[str, bytes]]) -> None:
values: Dict[str, bytes] = dict(key_value_pairs)
def mset(
self, key_value_pairs: Sequence[Tuple[str, Union[bytes, Document]]]
) -> None:
values = dict(key_value_pairs)
with self._make_sync_session() as session:
self._mdelete(list(values.keys()), session)
session.add_all(
[
LangchainKeyValueStores(namespace=self.namespace, key=k, value=v)
LangchainKeyValueStores(
namespace=self.namespace,
key=k,
value=self._bytes_or_document(v),
)
for k, v in values.items()
]
)
session.commit()

def _bytes_or_document(self, v: Union[bytes, Document]) -> Union[bytes, bytearray]:
if type(v) is bytes:
return v
elif type(v) is Document:
return bytearray(v.page_content, "utf8")
raise ValueError("Expecting bytes or langchain Document")

def _mdelete(self, keys: Sequence[str], session: Session) -> None:
stmt = delete(LangchainKeyValueStores).filter(
and_(
Expand Down
15 changes: 15 additions & 0 deletions libs/community/tests/unit_tests/storage/test_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,21 @@ def test_sample_sql_docstore(sql_store: SQLStore) -> None:
assert [key for key in sql_store.yield_keys()] == ["key2"]


@pytest.mark.xfail(is_sqlalchemy_v1, reason="SQLAlchemy 1.x issues")
def test_sample_sql_docstore_with_document(sql_store: SQLStore) -> None:
# Set values for keys
sql_store.mset([("key1", Document("value1")), ("key2", Document("value2"))])

# Get values for keys
values = sql_store.mget(["key1", "key2"])
assert values == [b"value1", b"value2"]
# Delete keys
sql_store.mdelete(["key1"])

# Iterate over keys
assert [key for key in sql_store.yield_keys()] == ["key2"]


@pytest.mark.requires("aiosqlite")
async def test_async_sample_sql_docstore(async_sql_store: SQLStore) -> None:
# Set values for keys
Expand Down
Loading