Merge pull request #112 from HKUDS/fix/grep-limit-overrun-merge-main

fix(grep): handle long rg lines without crashing

Author: tjb-tech

src/openharness/tools/grep_tool.py

@@ -183,6 +183,7 @@ async def _rg_grep(
         cwd=str(root),
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.PIPE,
+        limit=8 * 1024 * 1024,
     )
 
     matches: list[str] = []
@@ -239,6 +240,7 @@ async def _rg_grep_file(
         cwd=str(path.parent),
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.PIPE,
+        limit=8 * 1024 * 1024,
     )
 
     matches: list[str] = []
@@ -282,7 +284,10 @@ async def _collect_rg_matches(
 ) -> None:
     assert process.stdout is not None
     while len(matches) < limit:
-        raw = await process.stdout.readline()
+        try:
+            raw = await process.stdout.readline()
+        except ValueError:
+            continue
         if not raw:
             break
         line = raw.decode("utf-8", errors="replace").rstrip("\n")
@@ -300,7 +305,10 @@ async def _collect_rg_file_matches(
 ) -> None:
     assert process.stdout is not None
     while len(matches) < limit:
-        raw = await process.stdout.readline()
+        try:
+            raw = await process.stdout.readline()
+        except ValueError:
+            continue
         if not raw:
             break
         line = raw.decode("utf-8", errors="replace").rstrip("\n")

tests/test_tools/test_grep_tool.py

@@ -12,9 +12,20 @@ async def readline(self):
         return b""
 
 
-class _FakeProcess:
+class _ValueErrorThenEofStdout:
     def __init__(self):
-        self.stdout = _FakeStdout()
+        self.calls = 0
+
+    async def readline(self):
+        self.calls += 1
+        if self.calls == 1:
+            raise ValueError("Separator is not found, and chunk exceed the limit")
+        return b""
+
+
+class _FakeProcess:
+    def __init__(self, stdout=None):
+        self.stdout = stdout or _FakeStdout()
         self.stderr = None
         self.returncode = None
         self.terminated = False
@@ -54,3 +65,30 @@ async def fake_create_subprocess_exec(*args, **kwargs):
     assert result.is_error is True
     assert "grep timed out after 1 seconds" in result.output
     assert fake_process.terminated or fake_process.killed
+
+
+@pytest.mark.asyncio
+async def test_grep_tool_uses_large_stream_limit_and_skips_valueerror(monkeypatch, tmp_path: Path):
+    tool = GrepTool()
+    monkeypatch.setattr("openharness.tools.grep_tool.shutil.which", lambda _: "/usr/bin/rg")
+    fake_process = _FakeProcess(stdout=_ValueErrorThenEofStdout())
+    seen_kwargs = {}
+
+    async def fake_create_subprocess_exec(*args, **kwargs):
+        seen_kwargs.update(kwargs)
+        fake_process.returncode = 1
+        return fake_process
+
+    monkeypatch.setattr(
+        "openharness.tools.grep_tool.asyncio.create_subprocess_exec",
+        fake_create_subprocess_exec,
+    )
+
+    result = await tool.execute(
+        GrepToolInput(pattern="foo"),
+        type("Ctx", (), {"cwd": tmp_path})(),
+    )
+
+    assert result.is_error is False
+    assert result.output == "(no matches)"
+    assert seen_kwargs["limit"] == 8 * 1024 * 1024