Add custom source for user-provided JSONL, bump to v0.3.1

Users can drop pre-formatted JSONL files into ~/.dataclaw/custom/<project>/
and DataClaw picks them up alongside all other sources. Sessions are
validated for required fields and redacted through the standard pipeline.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: peteromallet

dataclaw/cli.py

@@ -12,7 +12,7 @@
 
 from .anonymizer import Anonymizer
 from .config import CONFIG_FILE, DataClawConfig, load_config, save_config
-from .parser import CLAUDE_DIR, CODEX_DIR, GEMINI_DIR, OPENCODE_DIR, OPENCLAW_DIR, discover_projects, parse_project_sessions
+from .parser import CLAUDE_DIR, CODEX_DIR, CUSTOM_DIR, GEMINI_DIR, OPENCODE_DIR, OPENCLAW_DIR, discover_projects, parse_project_sessions
 from .secrets import _has_mixed_char_types, _shannon_entropy, redact_session
 
 HF_TAG = "dataclaw"
@@ -58,8 +58,8 @@
     "Step 6/6: After explicit user approval, publish: dataclaw export --publish-attestation \"User explicitly approved publishing to Hugging Face.\"",
 ]
 
-EXPLICIT_SOURCE_CHOICES = {"claude", "codex", "gemini", "opencode", "openclaw", "all", "both"}
-SOURCE_CHOICES = ["auto", "claude", "codex", "gemini", "opencode", "openclaw", "all"]
+EXPLICIT_SOURCE_CHOICES = {"claude", "codex", "custom", "gemini", "opencode", "openclaw", "all", "both"}
+SOURCE_CHOICES = ["auto", "claude", "codex", "custom", "gemini", "opencode", "openclaw", "all"]
 
 
 def _mask_secret(s: str) -> str:
@@ -89,7 +89,9 @@ def _source_label(source_filter: str) -> str:
         return "OpenCode"
     if source_filter == "openclaw":
         return "OpenClaw"
-    return "Claude Code, Codex, Gemini CLI, OpenCode, or OpenClaw"
+    if source_filter == "custom":
+        return "Custom"
+    return "Claude Code, Codex, Gemini CLI, OpenCode, OpenClaw, or Custom"
 
 
 def _normalize_source_filter(source_filter: str) -> str:
@@ -133,7 +135,9 @@ def _has_session_sources(source_filter: str = "auto") -> bool:
         return OPENCODE_DIR.exists()
     if source_filter == "openclaw":
         return OPENCLAW_DIR.exists()
-    return CLAUDE_DIR.exists() or CODEX_DIR.exists() or GEMINI_DIR.exists() or OPENCODE_DIR.exists() or OPENCLAW_DIR.exists()
+    if source_filter == "custom":
+        return CUSTOM_DIR.exists()
+    return CLAUDE_DIR.exists() or CODEX_DIR.exists() or CUSTOM_DIR.exists() or GEMINI_DIR.exists() or OPENCODE_DIR.exists() or OPENCLAW_DIR.exists()
 
 
 def _filter_projects_by_source(projects: list[dict], source_filter: str) -> list[dict]:

dataclaw/parser.py

@@ -19,6 +19,7 @@
 GEMINI_SOURCE = "gemini"
 OPENCODE_SOURCE = "opencode"
 OPENCLAW_SOURCE = "openclaw"
+CUSTOM_SOURCE = "custom"
 
 CLAUDE_DIR = Path.home() / ".claude"
 PROJECTS_DIR = CLAUDE_DIR / "projects"
@@ -38,6 +39,8 @@
 OPENCLAW_AGENTS_DIR = OPENCLAW_DIR / "agents"
 UNKNOWN_OPENCLAW_CWD = "<unknown-cwd>"
 
+CUSTOM_DIR = Path.home() / ".dataclaw" / "custom"
+
 _CODEX_PROJECT_INDEX: dict[str, list[Path]] = {}
 _GEMINI_HASH_MAP: dict[str, str] = {}
 _OPENCODE_PROJECT_INDEX: dict[str, list[str]] = {}
@@ -129,6 +132,7 @@ def discover_projects() -> list[dict]:
     projects.extend(_discover_gemini_projects())
     projects.extend(_discover_opencode_projects())
     projects.extend(_discover_openclaw_projects())
+    projects.extend(_discover_custom_projects())
     return sorted(projects, key=lambda p: (p["display_name"], p["source"]))
 
 
@@ -246,13 +250,99 @@ def _discover_openclaw_projects() -> list[dict]:
     return projects
 
 
+def _discover_custom_projects() -> list[dict]:
+    if not CUSTOM_DIR.exists():
+        return []
+
+    projects = []
+    for project_dir in sorted(CUSTOM_DIR.iterdir()):
+        if not project_dir.is_dir():
+            continue
+        jsonl_files = list(project_dir.glob("*.jsonl"))
+        if not jsonl_files:
+            continue
+        session_count = 0
+        total_size = 0
+        for f in jsonl_files:
+            total_size += f.stat().st_size
+            try:
+                session_count += sum(1 for line in f.open() if line.strip())
+            except OSError:
+                pass
+        if session_count == 0:
+            continue
+        projects.append(
+            {
+                "dir_name": project_dir.name,
+                "display_name": f"custom:{project_dir.name}",
+                "session_count": session_count,
+                "total_size_bytes": total_size,
+                "source": CUSTOM_SOURCE,
+            }
+        )
+    return projects
+
+
+def _parse_custom_sessions(
+    project_dir_name: str,
+    anonymizer: Anonymizer,
+) -> list[dict]:
+    project_path = CUSTOM_DIR / project_dir_name
+    if not project_path.exists():
+        return []
+
+    required_fields = {"session_id", "model", "messages"}
+    sessions = []
+    for jsonl_file in sorted(project_path.glob("*.jsonl")):
+        try:
+            for line_num, line in enumerate(jsonl_file.open(), 1):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    session = json.loads(line)
+                except json.JSONDecodeError:
+                    logger.warning(
+                        "custom:%s: %s line %d: invalid JSON, skipping",
+                        project_dir_name, jsonl_file.name, line_num,
+                    )
+                    continue
+                if not isinstance(session, dict):
+                    logger.warning(
+                        "custom:%s: %s line %d: not a JSON object, skipping",
+                        project_dir_name, jsonl_file.name, line_num,
+                    )
+                    continue
+                missing = required_fields - session.keys()
+                if missing:
+                    logger.warning(
+                        "custom:%s: %s line %d: missing required fields %s, skipping",
+                        project_dir_name, jsonl_file.name, line_num, sorted(missing),
+                    )
+                    continue
+                session["project"] = f"custom:{project_dir_name}"
+                session["source"] = CUSTOM_SOURCE
+                # Redact message content through the anonymizer
+                for msg in session.get("messages", []):
+                    if "content" in msg and isinstance(msg["content"], str):
+                        redacted, _ = redact_text(msg["content"])
+                        msg["content"] = anonymizer.text(redacted)
+                sessions.append(session)
+        except OSError:
+            logger.warning("custom:%s: failed to read %s", project_dir_name, jsonl_file.name)
+    return sessions
+
+
 def parse_project_sessions(
     project_dir_name: str,
     anonymizer: Anonymizer,
     include_thinking: bool = True,
     source: str = CLAUDE_SOURCE,
 ) -> list[dict]:
     """Parse all sessions for a project into structured dicts."""
+    if source == CUSTOM_SOURCE:
+        return _parse_custom_sessions(project_dir_name, anonymizer)
+
     if source == OPENCLAW_SOURCE:
         index = _get_openclaw_project_index()
         session_files = index.get(project_dir_name, [])

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataclaw"
-version = "0.3.0"
+version = "0.3.1"
 description = "Export your coding agent conversations to Hugging Face as structured data"
 requires-python = ">=3.10"
 license = "MIT"

tests/test_cli.py

@@ -400,7 +400,7 @@ def test_no_projects(self, monkeypatch, capsys):
         monkeypatch.setattr("dataclaw.cli.discover_projects", lambda: [])
         list_projects()
         captured = capsys.readouterr()
-        assert "No Claude Code, Codex, Gemini CLI, OpenCode, or OpenClaw sessions" in captured.out
+        assert "No Claude Code, Codex, Gemini CLI, OpenCode, OpenClaw, or Custom sessions" in captured.out
 
     def test_source_filter_codex(self, monkeypatch, capsys):
         monkeypatch.setattr(
@@ -626,7 +626,7 @@ def test_export_requires_explicit_source_selection(self, monkeypatch, capsys):
         assert payload["error"] == "Source scope is not confirmed yet."
         assert payload["blocked_on_step"] == "Step 2/6"
         assert len(payload["process_steps"]) == 6
-        assert payload["allowed_sources"] == ["all", "both", "claude", "codex", "gemini", "openclaw", "opencode"]
+        assert payload["allowed_sources"] == ["all", "both", "claude", "codex", "custom", "gemini", "openclaw", "opencode"]
         assert payload["next_command"] == "dataclaw config --source all"
 
     def test_configure_next_steps_require_full_folder_presentation(self):

tests/test_parser.py

@@ -437,6 +437,7 @@ def _disable_codex(self, tmp_path, monkeypatch):
         monkeypatch.setattr("dataclaw.parser._OPENCODE_PROJECT_INDEX", {})
         monkeypatch.setattr("dataclaw.parser.OPENCLAW_AGENTS_DIR", tmp_path / "no-openclaw-agents")
         monkeypatch.setattr("dataclaw.parser._OPENCLAW_PROJECT_INDEX", {})
+        monkeypatch.setattr("dataclaw.parser.CUSTOM_DIR", tmp_path / "no-custom")
 
     def _write_opencode_db(self, db_path):
         conn = sqlite3.connect(db_path)
@@ -987,6 +988,7 @@ def _disable_codex(self, tmp_path, monkeypatch):
         monkeypatch.setattr("dataclaw.parser._OPENCODE_PROJECT_INDEX", {})
         monkeypatch.setattr("dataclaw.parser.OPENCLAW_AGENTS_DIR", tmp_path / "no-openclaw-agents")
         monkeypatch.setattr("dataclaw.parser._OPENCLAW_PROJECT_INDEX", {})
+        monkeypatch.setattr("dataclaw.parser.CUSTOM_DIR", tmp_path / "no-custom")
 
     def test_discover_includes_subagent_sessions(self, tmp_path, monkeypatch, mock_anonymizer):
         self._disable_codex(tmp_path, monkeypatch)
@@ -1585,6 +1587,7 @@ def _disable_others(self, tmp_path, monkeypatch):
         monkeypatch.setattr("dataclaw.parser.OPENCODE_DB_PATH", tmp_path / "no-opencode.db")
         monkeypatch.setattr("dataclaw.parser._OPENCODE_PROJECT_INDEX", {})
         monkeypatch.setattr("dataclaw.parser._OPENCLAW_PROJECT_INDEX", {})
+        monkeypatch.setattr("dataclaw.parser.CUSTOM_DIR", tmp_path / "no-custom")
 
     def test_discover_openclaw_projects(self, tmp_path, monkeypatch, mock_anonymizer):
         self._disable_others(tmp_path, monkeypatch)
@@ -1657,3 +1660,117 @@ def test_multiple_agents_same_cwd(self, tmp_path, monkeypatch, mock_anonymizer):
         projects = discover_projects()
         assert len(projects) == 1
         assert projects[0]["session_count"] == 2
+
+
+class TestDiscoverCustomProjects:
+    def _disable_others(self, tmp_path, monkeypatch):
+        monkeypatch.setattr("dataclaw.parser.PROJECTS_DIR", tmp_path / "no-claude")
+        monkeypatch.setattr("dataclaw.parser.CODEX_SESSIONS_DIR", tmp_path / "no-codex-sessions")
+        monkeypatch.setattr("dataclaw.parser.CODEX_ARCHIVED_DIR", tmp_path / "no-codex-archived")
+        monkeypatch.setattr("dataclaw.parser._CODEX_PROJECT_INDEX", {})
+        monkeypatch.setattr("dataclaw.parser.GEMINI_DIR", tmp_path / "no-gemini")
+        monkeypatch.setattr("dataclaw.parser.OPENCODE_DB_PATH", tmp_path / "no-opencode.db")
+        monkeypatch.setattr("dataclaw.parser._OPENCODE_PROJECT_INDEX", {})
+        monkeypatch.setattr("dataclaw.parser.OPENCLAW_AGENTS_DIR", tmp_path / "no-openclaw-agents")
+        monkeypatch.setattr("dataclaw.parser._OPENCLAW_PROJECT_INDEX", {})
+
+    def _make_valid_session(self, session_id="s1", model="gpt-4", content="hello"):
+        return json.dumps({
+            "session_id": session_id,
+            "model": model,
+            "messages": [
+                {"role": "user", "content": content},
+                {"role": "assistant", "content": "hi there"},
+            ],
+            "stats": {"user_messages": 1, "assistant_messages": 1, "tool_uses": 0,
+                       "input_tokens": 10, "output_tokens": 5},
+        })
+
+    def test_discover_custom_projects(self, tmp_path, monkeypatch, mock_anonymizer):
+        self._disable_others(tmp_path, monkeypatch)
+        custom_dir = tmp_path / "custom"
+        proj = custom_dir / "my-project"
+        proj.mkdir(parents=True)
+        (proj / "sessions.jsonl").write_text(
+            self._make_valid_session("s1") + "\n" + self._make_valid_session("s2") + "\n"
+        )
+        monkeypatch.setattr("dataclaw.parser.CUSTOM_DIR", custom_dir)
+        projects = discover_projects()
+        assert len(projects) == 1
+        assert projects[0]["display_name"] == "custom:my-project"
+        assert projects[0]["session_count"] == 2
+        assert projects[0]["source"] == "custom"
+
+    def test_discover_skips_empty_dir(self, tmp_path, monkeypatch):
+        self._disable_others(tmp_path, monkeypatch)
+        custom_dir = tmp_path / "custom"
+        (custom_dir / "empty-project").mkdir(parents=True)
+        monkeypatch.setattr("dataclaw.parser.CUSTOM_DIR", custom_dir)
+        projects = discover_projects()
+        assert len(projects) == 0
+
+    def test_discover_missing_dir(self, tmp_path, monkeypatch):
+        self._disable_others(tmp_path, monkeypatch)
+        monkeypatch.setattr("dataclaw.parser.CUSTOM_DIR", tmp_path / "nonexistent")
+        projects = discover_projects()
+        assert len(projects) == 0
+
+    def test_parse_valid_sessions(self, tmp_path, monkeypatch, mock_anonymizer):
+        custom_dir = tmp_path / "custom"
+        proj = custom_dir / "test-proj"
+        proj.mkdir(parents=True)
+        (proj / "data.jsonl").write_text(
+            self._make_valid_session("s1") + "\n" + self._make_valid_session("s2", model="o1") + "\n"
+        )
+        monkeypatch.setattr("dataclaw.parser.CUSTOM_DIR", custom_dir)
+        sessions = parse_project_sessions("test-proj", mock_anonymizer, source="custom")
+        assert len(sessions) == 2
+        assert sessions[0]["session_id"] == "s1"
+        assert sessions[1]["model"] == "o1"
+        assert sessions[0]["project"] == "custom:test-proj"
+        assert sessions[0]["source"] == "custom"
+
+    def test_parse_skips_missing_fields(self, tmp_path, monkeypatch, mock_anonymizer):
+        custom_dir = tmp_path / "custom"
+        proj = custom_dir / "test-proj"
+        proj.mkdir(parents=True)
+        valid = self._make_valid_session("s1")
+        no_model = json.dumps({"session_id": "s2", "messages": []})
+        no_messages = json.dumps({"session_id": "s3", "model": "m"})
+        no_session_id = json.dumps({"model": "m", "messages": []})
+        (proj / "data.jsonl").write_text(
+            "\n".join([valid, no_model, no_messages, no_session_id]) + "\n"
+        )
+        monkeypatch.setattr("dataclaw.parser.CUSTOM_DIR", custom_dir)
+        sessions = parse_project_sessions("test-proj", mock_anonymizer, source="custom")
+        assert len(sessions) == 1
+        assert sessions[0]["session_id"] == "s1"
+
+    def test_parse_skips_invalid_json(self, tmp_path, monkeypatch, mock_anonymizer):
+        custom_dir = tmp_path / "custom"
+        proj = custom_dir / "test-proj"
+        proj.mkdir(parents=True)
+        valid = self._make_valid_session("s1")
+        (proj / "data.jsonl").write_text(valid + "\n" + "not-json\n")
+        monkeypatch.setattr("dataclaw.parser.CUSTOM_DIR", custom_dir)
+        sessions = parse_project_sessions("test-proj", mock_anonymizer, source="custom")
+        assert len(sessions) == 1
+
+    def test_parse_multiple_files(self, tmp_path, monkeypatch, mock_anonymizer):
+        custom_dir = tmp_path / "custom"
+        proj = custom_dir / "test-proj"
+        proj.mkdir(parents=True)
+        (proj / "a.jsonl").write_text(self._make_valid_session("s1") + "\n")
+        (proj / "b.jsonl").write_text(self._make_valid_session("s2") + "\n")
+        monkeypatch.setattr("dataclaw.parser.CUSTOM_DIR", custom_dir)
+        sessions = parse_project_sessions("test-proj", mock_anonymizer, source="custom")
+        assert len(sessions) == 2
+        ids = {s["session_id"] for s in sessions}
+        assert ids == {"s1", "s2"}
+
+    def test_parse_nonexistent_project(self, tmp_path, monkeypatch, mock_anonymizer):
+        custom_dir = tmp_path / "custom"
+        custom_dir.mkdir(parents=True)
+        monkeypatch.setattr("dataclaw.parser.CUSTOM_DIR", custom_dir)
+        sessions = parse_project_sessions("nope", mock_anonymizer, source="custom")
+        assert sessions == []