[codex] Support env OAuth tokens and robust indexing

.env.example

@@ -5,6 +5,16 @@ X_CLIENT_ID=
 X_CLIENT_SECRET=
 X_REDIRECT_URI=http://127.0.0.1:8765/callback
 
+# Optional. If you already have OAuth token secrets, extraction can use them
+# directly instead of data/.secrets/x-token.json. X_TOKEN_JSON may contain the
+# full OAuth token response; otherwise set X_ACCESS_TOKEN and, if available,
+# X_REFRESH_TOKEN.
+# X_TOKEN_JSON=
+# X_ACCESS_TOKEN=
+# X_REFRESH_TOKEN=
+# X_EXPIRES_IN=
+# X_TOKEN_OBTAINED_AT=
+
 # Optional. Defaults shown here.
 PKB_DATA_DIR=data
 PKB_THREAD_SEARCH=all

README.md

@@ -50,6 +50,11 @@ data/
 
    If your X app is configured as a confidential client, also set `X_CLIENT_SECRET`.
 
+   If you already have OAuth token secrets, you can also put them in `.env` instead of
+   `data/.secrets/x-token.json`. Use `X_TOKEN_JSON` for the full OAuth token response,
+   or set `X_ACCESS_TOKEN` and optionally `X_REFRESH_TOKEN`, `X_EXPIRES_IN`, and
+   `X_TOKEN_OBTAINED_AT`.
+
 ## Install
 
 ```bash
@@ -112,7 +117,7 @@ There are two separate workflows:
    pkb extract
    ```
 
-   `pkb auth` briefly starts a local OAuth callback listener at `http://127.0.0.1:8765/callback`. After the browser-based authorization finishes, the token is saved at `data/.secrets/x-token.json`; no daemon needs to keep running.
+   `pkb auth` briefly starts a local OAuth callback listener at `http://127.0.0.1:8765/callback`. After the browser-based authorization finishes, the token is saved at `data/.secrets/x-token.json`; no daemon needs to keep running. If `.env` contains `X_TOKEN_JSON` or `X_ACCESS_TOKEN`, extraction can use that token directly; refreshed tokens are written to `data/.secrets/x-token.json`.
 
 2. Agent discovery:
 

src/pkb/config.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import os
+import json
 from dataclasses import dataclass
 from pathlib import Path
+from typing import Any
 
 
 DEFAULT_SCOPES = ("tweet.read", "users.read", "bookmark.read", "offline.access")
@@ -27,6 +29,7 @@ class Settings:
     client_secret: str | None
     redirect_uri: str
     data_dir: Path
+    env_token: dict[str, Any] | None = None
     scopes: tuple[str, ...] = DEFAULT_SCOPES
     thread_search: str = "all"
     link_timeout_seconds: float = 20.0
@@ -66,5 +69,37 @@ def load_settings() -> Settings:
         client_secret=client_secret,
         redirect_uri=redirect_uri,
         data_dir=data_dir,
+        env_token=_load_env_token(),
         thread_search=thread_search,
     )
+
+
+def _load_env_token() -> dict[str, Any] | None:
+    token_json = os.environ.get("X_TOKEN_JSON", "").strip()
+    if token_json:
+        token = json.loads(token_json)
+        if not isinstance(token, dict):
+            raise ValueError("X_TOKEN_JSON must be a JSON object")
+        return token
+
+    access_token = os.environ.get("X_ACCESS_TOKEN", "").strip() or os.environ.get("X_BEARER_TOKEN", "").strip()
+    if not access_token:
+        return None
+
+    token: dict[str, Any] = {
+        "access_token": access_token,
+        "token_type": os.environ.get("X_TOKEN_TYPE", "bearer").strip() or "bearer",
+    }
+    refresh_token = os.environ.get("X_REFRESH_TOKEN", "").strip()
+    if refresh_token:
+        token["refresh_token"] = refresh_token
+    scope = os.environ.get("X_TOKEN_SCOPE", "").strip()
+    if scope:
+        token["scope"] = scope
+    expires_in = os.environ.get("X_EXPIRES_IN", "").strip()
+    if expires_in:
+        token["expires_in"] = int(expires_in)
+    obtained_at = os.environ.get("X_TOKEN_OBTAINED_AT", "").strip()
+    if obtained_at:
+        token["obtained_at"] = int(obtained_at)
+    return token

src/pkb/index.py

@@ -172,6 +172,8 @@ def _iter_markdown_documents(markdown_dir: Path) -> list[MarkdownDocument]:
         if not root.exists():
             continue
         for path in sorted(root.rglob("*.md")):
+            if not path.is_file():
+                continue
             documents.append(_load_markdown_document(markdown_dir, path, kind))
     return documents
 

src/pkb/oauth.py

@@ -124,6 +124,17 @@ def load_token(path: Path) -> dict[str, Any]:
     return json.loads(path.read_text(encoding="utf-8"))
 
 
+def load_configured_token(settings: Settings) -> dict[str, Any]:
+    if settings.env_token:
+        return dict(settings.env_token)
+    if not settings.token_path.exists():
+        raise ValueError(
+            f"No OAuth token found. Run `pkb auth` to create {settings.token_path}, "
+            "or set X_TOKEN_JSON or X_ACCESS_TOKEN in .env."
+        )
+    return load_token(settings.token_path)
+
+
 def token_is_expired(token: dict[str, Any], skew_seconds: int = 120) -> bool:
     expires_in = int(token.get("expires_in", 0) or 0)
     obtained_at = int(token.get("obtained_at", 0) or 0)
@@ -160,7 +171,7 @@ def authenticate(settings: Settings, open_browser: bool = True) -> dict[str, Any
 
 
 def get_valid_token(settings: Settings) -> dict[str, Any]:
-    token = load_token(settings.token_path)
+    token = load_configured_token(settings)
     if token_is_expired(token) and token.get("refresh_token"):
         refreshed = refresh_token(settings, token["refresh_token"])
         if "refresh_token" not in refreshed:

tests/test_index_search.py

@@ -122,6 +122,26 @@ def test_reindex_indexes_bookmark_and_linked_page_rows(tmp_path):
     assert linked_page.exists()
 
 
+def test_reindex_ignores_markdown_named_directories(tmp_path):
+    settings = _settings(tmp_path)
+    _write_linked_page(
+        settings,
+        "jeffreys-skills.md",
+        "https---jeffreys-skills-md-8a60d8366c9d",
+        url="https://jeffreys-skills.md/",
+        title="Jeffrey's Skills",
+        body="# Jeffrey's Skills\n\nDirectory name ends with .md.\n",
+    )
+
+    stats = reindex(settings)
+
+    assert stats.scanned == 1
+    assert stats.indexed == 1
+    assert [hit.path for hit in browse(settings, kind="linked-page")] == [
+        "linked-pages/jeffreys-skills.md/https---jeffreys-skills-md-8a60d8366c9d.md"
+    ]
+
+
 def test_reindex_skips_unchanged_full_reindexes_and_removes_deleted_files(tmp_path):
     settings = _settings(tmp_path)
     bookmark = _write_bookmark(

tests/test_oauth.py

@@ -0,0 +1,56 @@
+import json
+
+import pytest
+
+from pkb.config import Settings, load_settings
+from pkb.oauth import get_valid_token
+
+
+def test_load_settings_builds_token_from_env(monkeypatch):
+    monkeypatch.setenv("X_CLIENT_ID", "client")
+    monkeypatch.setenv("X_ACCESS_TOKEN", "access")
+    monkeypatch.setenv("X_REFRESH_TOKEN", "refresh")
+    monkeypatch.setenv("X_EXPIRES_IN", "7200")
+    monkeypatch.setenv("X_TOKEN_OBTAINED_AT", "123")
+
+    settings = load_settings()
+
+    assert settings.env_token == {
+        "access_token": "access",
+        "token_type": "bearer",
+        "refresh_token": "refresh",
+        "expires_in": 7200,
+        "obtained_at": 123,
+    }
+
+
+def test_load_settings_accepts_token_json(monkeypatch):
+    monkeypatch.setenv("X_TOKEN_JSON", json.dumps({"access_token": "json-access", "refresh_token": "json-refresh"}))
+
+    settings = load_settings()
+
+    assert settings.env_token == {"access_token": "json-access", "refresh_token": "json-refresh"}
+
+
+def test_get_valid_token_uses_env_token_when_token_file_is_missing(tmp_path):
+    settings = Settings(
+        client_id="client",
+        client_secret=None,
+        redirect_uri="http://127.0.0.1:8765/callback",
+        data_dir=tmp_path,
+        env_token={"access_token": "env-access"},
+    )
+
+    assert get_valid_token(settings)["access_token"] == "env-access"
+
+
+def test_get_valid_token_explains_missing_token_configuration(tmp_path):
+    settings = Settings(
+        client_id="client",
+        client_secret=None,
+        redirect_uri="http://127.0.0.1:8765/callback",
+        data_dir=tmp_path,
+    )
+
+    with pytest.raises(ValueError, match="X_TOKEN_JSON or X_ACCESS_TOKEN"):
+        get_valid_token(settings)