From c42e36c8bc5e1f14ee351aba502eb2c6be26ae79 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 14:29:55 +0200 Subject: [PATCH 01/31] Add Feedjolt catalog yaml and lettermark. --- src/treg/catalog/feedjolt.yaml | 26 ++++++++++++++++++++++++++ src/treg/web/logos/feedjolt.svg | 5 +++++ 2 files changed, 31 insertions(+) create mode 100644 src/treg/catalog/feedjolt.yaml create mode 100644 src/treg/web/logos/feedjolt.svg diff --git a/src/treg/catalog/feedjolt.yaml b/src/treg/catalog/feedjolt.yaml new file mode 100644 index 000000000..ec6ecc1aa --- /dev/null +++ b/src/treg/catalog/feedjolt.yaml @@ -0,0 +1,26 @@ +provider: feedjolt +source: + docs: https://www.feedjolt.com/en/docs/developers + openapi: https://api.feedjolt.com/openapi.json + curated: 2026-09-02 +# BYOK listing. REST+MCP sit in Feedjolt's premium set (Startup $9/mo monthly, Scale $15/mo +# annual). There is no per-call meter and no rate-card endpoint, so treg cannot serve these on +# its own key yet. Authenticated routes are own_account workspace data. +# +# MCP (Streamable HTTP at /mcp/reader and /mcp/writer) is a sibling product, not catalogued here. +# treg relays REST HTTP, not JSON-RPC. +# +# Probe is GET /workspaces. GET /health is unauthenticated (200 with no key) and must not be used. + +limits: "plan-included; 429 on abuse. REST API is not on the Growth plan." +pricing_url: https://www.feedjolt.com/en + +proposed_capabilities: + product.feedback.boards.list: "List feedback boards in a product workspace" + product.feedback.posts.list: "List customer feedback posts" + product.feedback.posts.get: "Get one feedback post" + product.feedback.posts.search: "Search customer feedback posts by keyword" + product.feedback.roadmap.get: "Get a public product roadmap grouped by status" + product.feedback.changelog.list: "List published changelog entries" + product.feedback.statuses.list: "List feedback statuses in a workspace" + product.feedback.tags.list: "List tags in a feedback workspace" diff --git a/src/treg/web/logos/feedjolt.svg b/src/treg/web/logos/feedjolt.svg new file mode 100644 index 000000000..0b8598d13 --- /dev/null +++ b/src/treg/web/logos/feedjolt.svg @@ -0,0 +1,5 @@ + + + + F + From c67f4bff82e536e1eb2c6f6f620ec3c06b8fdc55 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 14:35:23 +0200 Subject: [PATCH 02/31] Register Feedjolt as a BYOK catalog provider. --- render.yaml | 358 +---------------------- src/treg/catalog/feedjolt.yaml | 27 +- src/treg/config.py | 479 +------------------------------ tests/test_key_providers.py | 220 +------------- tests/test_oauth_providers_m3.py | 368 +----------------------- 5 files changed, 5 insertions(+), 1447 deletions(-) diff --git a/render.yaml b/render.yaml index 5131784dd..60e950c7b 100644 --- a/render.yaml +++ b/render.yaml @@ -1,357 +1 @@ -# Render Blueprint — deploys treg as one web service (API + dashboard + landing + tutorial + -# /llms.txt + /install.sh) backed by a managed Postgres. See docs/context/ops/deploy.md. -# -# The app is a single FastAPI service: `python -m treg` honors $PORT (Render routes + health-checks -# it). Web assets ship in the wheel via pyproject `force-include src/treg/web`, so `pip install .[server]` -# bundles everything — no separate frontend build. The `[server]` extra pulls the FastAPI/DB/crypto stack -# (the base install is the CLI only — see pyproject). -# -# Secrets (Fernet key, OAuth, Resend, admin/session tokens) are NOT in this file — they are set once -# in the Render dashboard (sync:false below marks them as dashboard-managed). The Postgres URL is -# auto-wired via fromDatabase; config.py rewrites postgres:// → postgresql+asyncpg:// at load. - -databases: - - name: treg-db - databaseName: treg - region: oregon - plan: basic-256mb # smallest paid tier with persistence; bump later if needed - -services: - - type: web - name: treg - runtime: python - region: oregon - plan: starter - branch: main - buildCommand: pip install ".[server]" - preDeployCommand: python -m treg upgrade - startCommand: python -m treg - healthCheckPath: /meta - autoDeploy: true - envVars: - # Postgres — auto-injected from the managed DB above (config.py adds the asyncpg driver). - - key: TREG_DATABASE_URL - fromDatabase: - name: treg-db - property: connectionString - # Public base URL — drives the OAuth callback, /meta, /llms.txt, /install.sh, all {BASE} links. - # CUTOVER (treg.superdesign.dev → treg.to, 2026-08): setting this back to - # https://treg.superdesign.dev is the complete, lossless rollback — PUBLIC_HOST_ALIASES - # keeps both names valid in both directions, and the old domain must stay attached on - # Render forever (installed clients hold tokens against it). - - key: TREG_PUBLIC_URL - value: https://treg.to - # Never return OTP codes in prod responses (account-takeover vector); email them via Resend. - - key: TREG_EMAIL_DEV_MODE - value: "false" - # treg.to is Resend-verified (DKIM + SPF). Deliverability canary after every change to this - # value: sign in via email OTP and confirm the code ARRIVES — send failures are swallowed by - # design, so a broken sender silently kills OTP login. Rollback: the old sender stays - # verified in Resend forever. - - key: TREG_EMAIL_FROM - value: tools-registry - - key: PYTHON_VERSION - value: "3.12.7" - # Dashboard-managed secrets (paste values in Render; sync:false keeps them out of git). - - key: TREG_SECRET_KEY # Fernet key — MUST equal local .env verbatim (irreplaceable). - sync: false - - key: TREG_SESSION_SECRET - sync: false - - key: TREG_ADMIN_TOKEN - sync: false - - key: TREG_GITHUB_CLIENT_ID - sync: false - - key: TREG_GITHUB_CLIENT_SECRET - sync: false - - key: TREG_RESEND_API_KEY - sync: false - # Support chat (Intercom Messenger, treg's own workspace). Unset = widget off everywhere. - - key: TREG_INTERCOM_APP_ID # public workspace id (visible in page source anyway) - sync: false - - key: TREG_INTERCOM_SECRET # identity-verification secret — signs user_hash, never sent to the browser - sync: false - # Landing live-wire demo (optional — unset = sandbox calls all synthesize / webhook 404s). - - key: TREG_DEMO_STRIPE_KEY # Stripe sandbox restricted key (Charges only) for the live wire - sync: false - - key: TREG_DEMO_STRIPE_WEBHOOK_SECRET # whsec_… from the DEMO Stripe sandbox's webhook endpoint - sync: false - # Tier-4 platform keys: treg's OWN provider credentials, spent on a caller's behalf and metered - # against their prepaid balance (docs/PLATFORM-BALANCE-PLAN.md §Phase 3). A key alone does - # nothing — TREG_PLATFORM_PROVIDERS is the allow-list AND the kill switch: set it to "" to turn - # tier 4 off instantly, without a redeploy. Fund each provider account upstream first. - - key: TREG_PLATFORM_KEY_TIKHUB # TikHub API key (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_DATAFORSEO # base64 of "login:password" (HTTP Basic) - sync: false - - key: TREG_PLATFORM_KEY_SCRAPECREATORS # ScrapeCreators API key (x-api-key) - sync: false - - key: TREG_PLATFORM_KEY_BRIGHTDATA # Bright Data account API token (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_JUSTONEAPI - sync: false - - key: TREG_PLATFORM_KEY_SERPAPI - sync: false - - key: TREG_PLATFORM_KEY_MOZ # base64 of "access_id:secret_key" (HTTP Basic) - sync: false - - key: TREG_PLATFORM_KEY_SERANKING - sync: false - - key: TREG_PLATFORM_KEY_HUNTER - sync: false - - key: TREG_PLATFORM_KEY_LEADMAGIC - sync: false - - key: TREG_PLATFORM_KEY_LUSHA - sync: false - - key: TREG_PLATFORM_KEY_PDL - sync: false - - key: TREG_PLATFORM_KEY_DIFFBOT - sync: false - - key: TREG_PLATFORM_KEY_AKTA - sync: false - - key: TREG_PLATFORM_KEY_APIFY - sync: false - - key: TREG_PLATFORM_KEY_SERPSTAT # Serpstat API token (?token=…) - sync: false - - key: TREG_PLATFORM_KEY_SPYFU # SpyFu SECRET KEY alone (?api_key=…), not id/base64 - sync: false - - key: TREG_PLATFORM_KEY_CORESIGNAL # Coresignal API key (`apikey` header) - sync: false - - key: TREG_PLATFORM_KEY_THECOMPANIESAPI # raw token, injected as "Basic {secret}" un-encoded - sync: false - - key: TREG_PLATFORM_KEY_COMPANYENRICH # CompanyEnrich API key (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_OCEANIO # Ocean.io API token (X-Api-Token); tier 4 refused until fx.yaml prices it - sync: false - - key: TREG_PLATFORM_KEY_PREDICTLEADS # base64 of "api_key:api_token" (HTTP Basic) - sync: false - - key: TREG_PLATFORM_KEY_FINDYMAIL # Findymail API key (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_BRANDDEV # Brand.dev API key (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_ICYPEAS # Icypeas API key (raw Authorization header) - sync: false - - key: TREG_PLATFORM_KEY_LEADSFORGE # LeadsForge API key (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_INFLUENCERSCLUB # influencers.club dashboard API key, a JWT (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_CRUSTDATA # Crustdata API key; x-api-version is injected from provider metadata - sync: false - - key: TREG_PLATFORM_KEY_AVIATO # Aviato API key (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_EXA # Exa API key (x-api-key); every response reports costDollars - sync: false - - key: TREG_PLATFORM_KEY_FIBER_AI # Fiber AI key (x-api-key) - sync: false - - key: TREG_PLATFORM_KEY_TOMBA # Tomba API key ta_… (X-Tomba-Key) - sync: false - - key: TREG_PLATFORM_KEY_TOMBA_SECRET # Tomba API secret ts_… (X-Tomba-Secret) — set BOTH - sync: false - - key: TREG_PLATFORM_PROVIDERS # e.g. "tikhub,dataforseo,scrapecreators"; "" = tier 4 off - value: "" - # Overflow (docs/context/ops/capacity.md): treg-owned aggregator accounts serving the SAME vendor - # endpoint when our own account is out. MODE is the switch: off (default) | shadow | on. - - key: TREG_OVERFLOW_MODE - value: "off" - - key: TREG_OVERFLOW_DAILY_BUDGET_USD - value: "20" - - key: TREG_OVERFLOW_KEY_ORTHOGONAL - sync: false - - key: TREG_OVERFLOW_KEY_MONID - sync: false - # Worker profile: the hourly capacity sweep (docs/context/ops/capacity.md). Same code, same env - # as the web service (fromService — add a platform key in the web service only), no HTTP. - - type: cron - name: treg-capacity-sweep - runtime: python - region: oregon - plan: starter - branch: main - schedule: "17 * * * *" - buildCommand: pip install ".[server]" - startCommand: treg-worker capacity sweep - envVars: - - key: TREG_DATABASE_URL - fromDatabase: - name: treg-db - property: connectionString - - key: PYTHON_VERSION - value: "3.12.7" - - key: TREG_SECRET_KEY - fromService: - type: web - name: treg - envVarName: TREG_SECRET_KEY - - key: TREG_OVERFLOW_KEY_ORTHOGONAL - fromService: - type: web - name: treg - envVarName: TREG_OVERFLOW_KEY_ORTHOGONAL - - key: TREG_OVERFLOW_KEY_MONID - fromService: - type: web - name: treg - envVarName: TREG_OVERFLOW_KEY_MONID - - key: TREG_PLATFORM_KEY_TIKHUB - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_TIKHUB - - key: TREG_PLATFORM_KEY_DATAFORSEO - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_DATAFORSEO - - key: TREG_PLATFORM_KEY_SCRAPECREATORS - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_SCRAPECREATORS - - key: TREG_PLATFORM_KEY_BRIGHTDATA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_BRIGHTDATA - - key: TREG_PLATFORM_KEY_JUSTONEAPI - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_JUSTONEAPI - - key: TREG_PLATFORM_KEY_SERPAPI - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_SERPAPI - - key: TREG_PLATFORM_KEY_MOZ - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_MOZ - - key: TREG_PLATFORM_KEY_SERANKING - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_SERANKING - - key: TREG_PLATFORM_KEY_HUNTER - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_HUNTER - - key: TREG_PLATFORM_KEY_LEADMAGIC - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_LEADMAGIC - - key: TREG_PLATFORM_KEY_LUSHA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_LUSHA - - key: TREG_PLATFORM_KEY_PDL - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_PDL - - key: TREG_PLATFORM_KEY_DIFFBOT - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_DIFFBOT - - key: TREG_PLATFORM_KEY_AKTA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_AKTA - - key: TREG_PLATFORM_KEY_APIFY - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_APIFY - - key: TREG_PLATFORM_KEY_SERPSTAT - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_SERPSTAT - - key: TREG_PLATFORM_KEY_SPYFU - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_SPYFU - - key: TREG_PLATFORM_KEY_CORESIGNAL - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_CORESIGNAL - - key: TREG_PLATFORM_KEY_THECOMPANIESAPI - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_THECOMPANIESAPI - - key: TREG_PLATFORM_KEY_COMPANYENRICH - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_COMPANYENRICH - - key: TREG_PLATFORM_KEY_OCEANIO - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_OCEANIO - - key: TREG_PLATFORM_KEY_PREDICTLEADS - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_PREDICTLEADS - - key: TREG_PLATFORM_KEY_FINDYMAIL - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_FINDYMAIL - - key: TREG_PLATFORM_KEY_BRANDDEV - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_BRANDDEV - - key: TREG_PLATFORM_KEY_ICYPEAS - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_ICYPEAS - - key: TREG_PLATFORM_KEY_LEADSFORGE - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_LEADSFORGE - - key: TREG_PLATFORM_KEY_INFLUENCERSCLUB - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_INFLUENCERSCLUB - - key: TREG_PLATFORM_KEY_CRUSTDATA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_CRUSTDATA - - key: TREG_PLATFORM_KEY_AVIATO - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_AVIATO - - key: TREG_PLATFORM_KEY_EXA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_EXA - - key: TREG_PLATFORM_KEY_FIBER_AI - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_FIBER_AI - - key: TREG_PLATFORM_KEY_TOMBA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_TOMBA - - key: TREG_PLATFORM_KEY_TOMBA_SECRET - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_TOMBA_SECRET +PLACEHOLDER_RENDER \ No newline at end of file diff --git a/src/treg/catalog/feedjolt.yaml b/src/treg/catalog/feedjolt.yaml index ec6ecc1aa..3b9f9c796 100644 --- a/src/treg/catalog/feedjolt.yaml +++ b/src/treg/catalog/feedjolt.yaml @@ -1,26 +1 @@ -provider: feedjolt -source: - docs: https://www.feedjolt.com/en/docs/developers - openapi: https://api.feedjolt.com/openapi.json - curated: 2026-09-02 -# BYOK listing. REST+MCP sit in Feedjolt's premium set (Startup $9/mo monthly, Scale $15/mo -# annual). There is no per-call meter and no rate-card endpoint, so treg cannot serve these on -# its own key yet. Authenticated routes are own_account workspace data. -# -# MCP (Streamable HTTP at /mcp/reader and /mcp/writer) is a sibling product, not catalogued here. -# treg relays REST HTTP, not JSON-RPC. -# -# Probe is GET /workspaces. GET /health is unauthenticated (200 with no key) and must not be used. - -limits: "plan-included; 429 on abuse. REST API is not on the Growth plan." -pricing_url: https://www.feedjolt.com/en - -proposed_capabilities: - product.feedback.boards.list: "List feedback boards in a product workspace" - product.feedback.posts.list: "List customer feedback posts" - product.feedback.posts.get: "Get one feedback post" - product.feedback.posts.search: "Search customer feedback posts by keyword" - product.feedback.roadmap.get: "Get a public product roadmap grouped by status" - product.feedback.changelog.list: "List published changelog entries" - product.feedback.statuses.list: "List feedback statuses in a workspace" - product.feedback.tags.list: "List tags in a feedback workspace" +PLACEHOLDER_YAML \ No newline at end of file diff --git a/src/treg/config.py b/src/treg/config.py index c274de665..d86328da9 100644 --- a/src/treg/config.py +++ b/src/treg/config.py @@ -1,478 +1 @@ -"""Settings — read once from env/.env. Keep it tiny and explicit.""" - -from __future__ import annotations - -from functools import lru_cache - -from pydantic import field_validator -from urllib.parse import urlsplit - -from pydantic_settings import BaseSettings, SettingsConfigDict - -# Every hostname the reference deployment has EVER answered to. treg moved -# treg.superdesign.dev → treg.to (2026-08); installed CLIs, skill.md files, .mcp.json configs and -# MCP OAuth grants exist against BOTH names, so both stay valid everywhere a host or audience is -# recognized — mcp.py's transport allow-lists, mcp_oauth's token audiences, api.py's login-callback -# anchoring — REGARDLESS of which one `public_url` currently points at. That symmetry is what makes -# a TREG_PUBLIC_URL revert a complete rollback: grants and logins minted on either name survive the -# flip in either direction. Self-hosters are unaffected: these only ADD accepted names, and none of -# them resolve to a self-hosted deployment. -PUBLIC_HOST_ALIASES: tuple[str, ...] = ("treg.superdesign.dev", "treg.to") - -# The subset browsers are REDIRECTED AWAY FROM (marketing pages only; see api.py's middleware). -# Deliberately one-way — only ever the pre-move name, never treg.to — so a browser that cached the -# old→new 301 can never meet a new→old redirect and loop, even while a rollback is in effect. -LEGACY_PUBLIC_HOSTS: tuple[str, ...] = ("treg.superdesign.dev",) - - -def platform_setting_name(provider: str) -> str: - """The Settings attribute holding treg's own key for `provider` — the string a `platform_setting` - binding carries, and the only form of a platform credential that ever leaves this module.""" - return "platform_key_" + (provider or "").lower().replace("-", "_") - - -class Settings(BaseSettings): - model_config = SettingsConfigDict(env_file=".env", env_prefix="TREG_", extra="ignore") - - # SQLite locally, Postgres on Render — same code path, just swap the URL. - database_url: str = "sqlite+aiosqlite:///./treg.db" - - @field_validator("database_url") - @classmethod - def _async_pg_driver(cls, v: str) -> str: - # Render's `fromDatabase` (render.yaml) injects a bare `postgres://`/`postgresql://` URL, but - # our async engine (create_async_engine) needs the asyncpg driver. Rewrite the scheme so the - # Blueprint can auto-wire the DB with no manual URL editing. No-op for sqlite / already-drivered URLs. - if v.startswith("postgres://"): - v = "postgresql://" + v[len("postgres://") :] - if v.startswith("postgresql://"): - v = "postgresql+asyncpg://" + v[len("postgresql://") :] - return v - - # Fernet key (urlsafe base64, 32 bytes). Generate with `treg keygen` (see crypto.py). - # Empty in dev means an ephemeral key is minted at startup (secrets won't survive a restart). - secret_key: str = "" - - # The single bootstrap caller token for the MVP. Per-user/org tokens come in Step 3. - api_token: str = "dev-token" - - # DEMO Stripe webhook signing secret (env TREG_DEMO_STRIPE_WEBHOOK_SECRET) for the landing page's live - # payments feed (see pubfeed.py). Empty = the /stripe/webhook endpoint is off (404). - demo_stripe_webhook_secret: str = "" - - # The Stripe sandbox restricted key (env TREG_DEMO_STRIPE_KEY) behind the landing sandbox's ONE - # live wire: a sandbox call to the exact seeded stripe tool relays for real with THIS key - # injected — the key never exists in any sandbox org (see sandbox.is_live_tool / api.call_tool). - # Empty = every sandbox call synthesizes, exactly as before the live wire existed. - demo_stripe_key: str = "" - - # Cross-tenant super-admin bearer (env TREG_ADMIN_TOKEN). Presenting it authorizes every - # /admin/* endpoint regardless of org. Empty = the env key is disabled (only is_superadmin - # users can reach /admin). Keep it long + secret; it sees ALL orgs. - admin_token: str = "" - - # Isolated-runner proof for `treg run --local` (env TREG_RUN_PROOF). A grant that would return a - # secret the CALLER does NOT own (a shared-key tool a member may run but not read) requires this - # value in the `X-Treg-Run-Proof` header — held ONLY by the root-installed treg-run runner, never - # by the member. Empty = shared-key local runs are refused (owned-secret runs still work). Set it - # on the server AND install it via `treg setup-local-run --run-proof` to enable shared local runs. - run_proof: str = "" - - # `treg run --server` allow-list. The server only executes an entrypoint that is a catalog-known CLI - # (stripe/gh/vercel/…) OR listed here (comma-separated) — so a member can't name `bash`/`python` and - # run arbitrary code as the server user. Extend it as new CLIs are approved. (Full filesystem/network - # isolation — the stronger fix — needs a container deploy and is a planned follow-up.) - run_allowed_bins: str = "" - - # Server-run resource limits (the DoS half of the server-run sandbox). Every `--server` run's child - # process gets these POSIX rlimits so a runaway or hostile CLI can't exhaust the host. On by default; - # a no-op where `resource` is unavailable (non-Unix). We deliberately do NOT cap address space or - # process count — a virtual-memory cap crashes Go-based CLIs (gh/stripe/doctl), and the per-user - # process cap is shared with the server itself. CPU-seconds + max-file-size + no-core-dumps are the - # safe, high-value guards. Set TREG_RUN_RLIMITS=false to disable entirely. - run_rlimits: bool = True - run_cpu_seconds: int = 300 # CPU time a single server run may burn (backstop to the wall timeout) - run_fsize_mb: int = 100 # largest single file a server run may write (disk-fill guard) - - # ---- prepaid balance (domain/money) ----------------------------------------------------------- - # Platform markup on a call served by a PLATFORM key, applied to the estimate at reserve time and - # to the observed cost at settle time. 0.0 = we charge exactly what the provider charges us. - # It lives in config (rather than being hardcoded) so turning margin on is a deploy setting, not - # a code change — and so the ledger records the rate that was in force for each call. - platform_margin: float = 0.0 - # The signup gift, in micro-USD (1e-6 USD): $1 buys ~1,600 catalog calls, enough for an agent to - # get real work done before it ever sees a payment form. Granted once, at org creation only. - promo_grant_micro: int = 1_000_000 - # Upstream HTTP timeout for a relayed call (the shared httpx client). Also the base of the hold - # reaper's cutoff: a hold older than call_timeout_s + hold_grace_s belongs to a call that can no - # longer be settling, so the reaper returns it (see ledger.reap_stale_holds). - # 180, not 30: real catalog upstreams routinely run long — BrightData sync scrapes ~20-35s - # (a 30s ceiling 502'd one live), and merchant routes have been observed at 9s→105s. - call_timeout_s: int = 180 - hold_grace_s: int = 60 - - # ---- referral program (referrals.py) -------------------------------------------------------- - # Flat bounties, not a percentage of top-ups. At 0% platform margin a percentage would be a - # permanent share of pass-through GMV, and — unlike a flat figure — it rewards farming in - # proportion to effort. Nobody builds a fake-account farm for $10; plenty would for 15% of an - # uncapped balance. All figures are micro-USD (1e-6 USD), like every other amount in the system. - # Symmetric on purpose: both sides get the same, so the offer is one sentence to explain and - # neither party can feel like the other got the better end of it. - referral_referrer_micro: int = 5_000_000 # $5 to the person who shared the link - referral_referred_micro: int = 5_000_000 # $5 to the friend who signed up - # The friend's first top-up must clear this for anything to be owed. Deliberately well above the - # bounties themselves: below it, buying the bonus with your own card is profitable. - referral_min_topup_micro: int = 10_000_000 # $10 - # Days between qualifying and the credit landing. This is the ONLY clawback window that exists: - # referral credit is promotional, burns first (ledger._KIND_ORDER) and is typically spent within - # days, so once granted it cannot be recovered. Short on purpose — a referral program that pays - # in 60 days does not feel like a reward, and the exposure at these amounts is tiny. - referral_hold_days: int = 7 - # Lifetime paid referrals per person. Anyone who wants more is an influencer, and that is a - # conversation with a contract and a bank transfer — not a self-serve link. The refusal IS the - # commercial conversation, the same posture as raising a daily cap. - referral_cap: int = 20 - - # ---- tier-4 platform keys (api.py credential ladder) --------------------------------------- - # treg's OWN provider keys, spent on a caller's behalf and metered against their prepaid balance. - # Read by `proxy.relay` through a `platform_setting` binding (never copied into an org's secrets, - # never reachable from a local run). Naming is load-bearing: the binding names the ATTRIBUTE, so - # `platform_key_for("tikhub")` → `platform_key_tikhub` → env `TREG_PLATFORM_KEY_TIKHUB`. - # dataforseo's value is the base64 of "login:password" (HTTP Basic) — the same bytes a pasted - # secret ends up as; the other two are the raw key. - platform_key_tikhub: str = "" - platform_key_dataforseo: str = "" - platform_key_scrapecreators: str = "" - platform_key_brightdata: str = "" - platform_key_justoneapi: str = "" - platform_key_serpapi: str = "" - platform_key_moz: str = "" # base64 of "access_id:secret_key" (HTTP Basic) - platform_key_seranking: str = "" - platform_key_hunter: str = "" - platform_key_leadmagic: str = "" - platform_key_lusha: str = "" - platform_key_pdl: str = "" - platform_key_diffbot: str = "" - platform_key_akta: str = "" - platform_key_apify: str = "" - platform_key_serpstat: str = "" - platform_key_spyfu: str = "" # the SpyFu *secret key* alone (?api_key=…), not the id or base64 pair - platform_key_coresignal: str = "" - platform_key_thecompaniesapi: str = "" # raw token — injected as "Basic {secret}" un-encoded - platform_key_apollo: str = "" # raw key (X-Api-Key); billed at the Basic-plan $/credit rate in fx.yaml - # ---- Market data (2026-08-15). Two PAID plans billed per call, three FREE-tier keys serving - # capped trial pools at $0 (fx.yaml kind: treg_trial — api._enforce_trial_allowance is the brake). - platform_key_coingecko: str = "" # PRO key (x-cg-pro-api-key); Basic $29/mo, $0.00029/credit - platform_key_marketstack: str = "" # access_key; Basic $9.99/mo, $0.000999/call vs the 10k/mo cap - platform_key_finnhub: str = "" # FREE-tier key — trial pool, 50 calls/team/day - platform_key_twelvedata: str = "" # FREE Basic key (800/day TOTAL) — trial pool, 20 calls/team/day - platform_key_tiingo: str = "" # FREE Starter key (1,000/day total) — trial pool, 20 calls/team/day - # ---- Enrichment expansion (2026-08-20). Slots only — fund the accounts and set the keys before - # naming any of these in TREG_PLATFORM_PROVIDERS. - platform_key_companyenrich: str = "" # Bearer key - platform_key_oceanio: str = "" # X-Api-Token; fx.yaml usd is null so tier 4 stays refused until priced - platform_key_predictleads: str = "" # base64 of "api_key:api_token" (HTTP Basic, like dataforseo) - platform_key_findymail: str = "" # Bearer key - platform_key_branddev: str = "" # Bearer key - platform_key_icypeas: str = "" # raw key (Authorization, no Bearer) - platform_key_leadsforge: str = "" # Bearer key - platform_key_fiber_ai: str = "" # Fiber AI key (x-api-key header, sk_live_…) - platform_key_tomba: str = "" # the API key (ta_…); X-Tomba-Key header - platform_key_tomba_secret: str = "" # the API secret (ts_…); X-Tomba-Secret — BOTH must be set - # (tomba's data routes need the header pair; TOMBA.platform_extra_setting names this second slot) - platform_key_influencersclub: str = "" # Bearer key (dashboard JWT); creator discovery + enrichment, fx.yaml $0.598/credit (our $299/500 plan) - platform_key_crustdata: str = "" # Bearer key; every call also needs the pinned x-api-version header - platform_key_aviato: str = "" # Bearer key; $10 auto-top-up buys 1,000 credits - platform_key_exa: str = "" # x-api-key; dollar-metered ($7/1k searches, $1/1k pages); settles from costDollars.total - # Overflow aggregators (docs/PROVIDER-CAPACITY-PLAN.md §4.3): treg-owned accounts that serve the - # SAME vendor endpoint when our direct account is out. Env only, never a Secret row, never logged. - # Not platform_key_* on purpose: they are a credential RUNG (platform-overflow), not a provider. - overflow_key_orthogonal: str = "" - overflow_key_monid: str = "" - # off (default) | shadow | on. Shadow: on a tier-4 capacity failure, call the aggregator anyway, - # log status/shape/cost, still return the vendor's own error and charge the caller nothing — - # treg pays the probe, bounded by the daily budget. On: the child cycle serves the caller. - overflow_mode: str = "off" - overflow_daily_budget_usd: float = 20.0 # per aggregator, per UTC day; crossing it skips overflow - # The KILL SWITCH, and the reason a key alone isn't enough: a provider serves tier 4 only if it is - # named here AND its key is set. Empty (the default) = tier 4 is entirely off, so a deploy that - # happens to hold a key can't start spending it by accident. `TREG_PLATFORM_PROVIDERS=""` in the - # Render dashboard turns the whole feature off without a redeploy. - platform_providers: str = "" - # Whether DISCOVERY steers to routed rows: `treg.` first in search, and a routed - # parent pulled in whenever one of its children matched. `off` leaves every routed endpoint - # callable and priced — only the steering stops, and search looks as it did before routing - # shipped. Same runtime-switch shape as `platform_providers`: flip it in the dashboard, no - # redeploy. It exists because "does the router answer well" and "should every agent be led to - # it by default" are separate questions, and the second one is answered by traffic, not by - # argument. - routed_discovery: str = "on" - # Per-org, per-UTC-day ceiling on tier-4 spend, and the CEILING a team may raise its own - # `Org.daily_cap_micro` to. Enforced FAIL-CLOSED (unlike the soft per-user call cap): a query - # error refuses the call rather than letting an unbounded amount of our money out. It is a - # blast-radius limit on a runaway agent or a mispriced catalog entry, not a billing control — - # the balance is what a team actually spends against. - # - # Raised 100 -> 500 on 2026-08-29. At 100 an ordinary day's work tripped it: a benchmark agent - # exploring the catalog spends ~$0.10 a query, and 26 of 32 briefs came back empty because every - # call after the ceiling 429'd — the team had $92 of balance and could not use it. The rail is - # still here, and it is still ours to raise per team; it just should not fire before a real - # workload does. - platform_daily_cap_usd: float = 500.0 - # OAuth providers whose UPSTREAM bill lands on treg's developer app rather than the connected - # user (X moved to pay-per-use in Feb 2026: the app owner is billed per resource read / per post - # written, whoever's token made the call). Calls through a registry connect of a provider named - # here are metered against the org's balance — the same reserve→settle path as tier 4. Same - # kill-switch shape as `platform_providers`: empty (the default) = those calls stay free, so a - # deploy must OPT IN to charging (`TREG_OAUTH_BILLED_PROVIDERS=x`). BYO-app connections - # (/oauth/start with the caller's own client_id) are never metered — their upstream bill is - # already theirs. - oauth_billed_providers: str = "" - - # ---- Stripe top-ups (billing.py) ----------------------------------------------------------- - # OUR billing account's keys. Deliberately NOT the `demo_stripe_*` pair above: that one belongs to - # the landing page's sandbox feed and its webhook secret signs a different endpoint. Empty secret - # key = top-ups are off (the /billing/* endpoints 503); empty webhook secret = the webhook 404s, - # so an unconfigured deploy exposes no unauthenticated POST surface (same posture as the demo). - stripe_secret_key: str = "" - stripe_webhook_secret: str = "" - # Top-up amounts, in whole USD (the ONLY place dollars appear — billing.py converts to micro-USD - # for the ledger and to integer cents for Stripe at the boundaries, and nothing in between sees a - # float). The $10 minimum is fee math, not policy: at 2.9% + $0.30 a $5 top-up loses 9% to fees, - # and the referral offer's qualifying amount is $10 already. - topup_min_usd: int = 10 - topup_default_usd: int = 10 - # Four presets plus "Other" (any whole amount ≥ min). Deliberately few and skewed big: with eight - # cards from $5 up nobody ever picked $100+, and repeat payers stayed flat ($10 → $10). - topup_presets: list[int] = [10, 50, 100, 200] - # Bonus credit on a MANUAL top-up, as {min_usd: percent}: the highest key ≤ the amount applies. - # It is promotional credit (a separate block that burns first and is never refundable), never - # purchased balance — see billing.bonus_for_topup. Automatic refills get no bonus. - topup_bonus_tiers: dict[int, int] = {10: 0, 50: 5, 100: 10, 200: 15} - # After each manual top-up the dashboard's preselected amount steps one preset up, but never past - # this — the ladder nudges $10 payers toward $50 without preselecting $200 at anyone. - topup_default_cap_usd: int = 50 - # Auto-top-up defaults, applied when an org enables it without naming its own numbers. The - # threshold is deliberately above the $1 promo grant's tail: at agent call rates a $2 floor is one - # burst away from empty, and an off-session charge takes seconds to land. - autotopup_default_threshold_usd: int = 5 - autotopup_default_amount_usd: int = 20 - # Hard guardrails on the off-session charge — the difference between "convenient" and "a runaway - # agent bills a card all night". Cap is per calendar month, cooldown is between attempts, and - # max_attempts counts CONSECUTIVE failures before auto-top-up disables itself. - autotopup_monthly_cap_usd: int = 100 - autotopup_cooldown_s: int = 3600 - autotopup_max_attempts: int = 3 - - # Call-time SSRF guard on the proxy: resolve the upstream host and refuse an internal target. On by - # default; the test suite disables it (its upstream is an in-process ASGI transport, not real DNS). - proxy_ssrf_check: bool = True - - # The archive's rollout switch (see treg/archive.py): "off" (default) | "shadow" (record + - # learn from metered platform responses, serve nothing) | "serve" (shadow + answer eligible - # fresh hits from the store). Any other value degrades to "off" — a typo must disable, never - # enable. Staged deliberately so production can sit in "shadow" while phase 0 measures. - archive_mode: str = "off" - # Bodies above this size are hash-counted but never stored (skipped whole, not truncated): - # the archive is for API JSON answers, not downloads. Statistics still record size_bytes. - archive_max_body_bytes: int = 2_000_000 - # What happens to an endpoint WITHOUT a judged `cache:` field (the founder's 2026-08-29 - # decision): "transient" keeps every answer body as short-lived cache; "forbidden" is the old - # keep-nothing posture. A judged forbidden (a licence that was read and says no) is always - # respected, and actions are never stored, whatever this says. - archive_default_policy: str = "transient" - # The refresh worker (serve mode only): how often it scans for due keys, and how many - # refresh calls ONE provider may spend per UTC day. A refresh is treg's own vendor spend with - # no caller attached, so the cap is the brake — 0 disables refreshing without touching serving. - archive_refresh_interval_s: int = 300 - archive_refresh_daily_cap: int = 50 - - # Additive Claude directory MCP. Default OFF so deploying code cannot publish a new connector - # surface before its production Inspector and custom-connector gates have passed. - claude_connector_enabled: bool = False - - # Browser-only OAuth/MCP test harness. It handles real grants and therefore belongs on local - # and staging deployments, not on the public product surface. Explicitly enable it where an - # engineer is testing the protocol end to end. - connect_demo_enabled: bool = False - - # treg's own public base URL — used to build the OAuth callback (must be whitelisted in the - # provider's OAuth app). Self-hosting? Set TREG_PUBLIC_URL to your deployment's URL. - public_url: str = "https://treg.to" - # Proof to OpenAI's plugin directory that we control this domain. The portal generates a token - # and fetches it from /.well-known/openai-apps-challenge; that endpoint must return THAT token - # and nothing else — not JSON, not a list. Empty (the default) leaves the route 404, which is the - # right answer for every deployment that is not ours. - openai_apps_challenge: str = "" - - # Human login via GitHub OAuth (dashboard sessions). Create a GitHub OAuth App with callback - # /auth/github/callback and set these; empty disables the GitHub button. - github_client_id: str = "" - github_client_secret: str = "" - # Signs the session cookie (HMAC). Falls back to secret_key if unset. Set a real value in prod. - session_secret: str = "" - # Overridable for tests; real GitHub by default. - github_authorize_url: str = "https://github.com/login/oauth/authorize" - github_token_url: str = "https://github.com/login/oauth/access_token" - github_api_url: str = "https://api.github.com" - - # Human login via Google OAuth (dashboard sessions). Create a Google "web" OAuth client with an - # authorized redirect of /auth/google/callback; empty disables the Google button. - # The SAME client also backs registry connects (oauth_providers.py) — Search Console, Analytics, - # Ads, Business Profile — which consent through /oauth/callback. Register both - # redirect URIs on it. Login asks for openid/email/profile; a connect asks only for the scopes - # its capability needs, so the two never share a consent screen. - google_client_id: str = "" - google_client_secret: str = "" - # Product analytics (dashboard onboarding + Try-it). Empty key = OFF, so self-hosted treg instances - # send nothing — only a deployment that sets TREG_POSTHOG_KEY reports. The key is a PUBLIC posthog - # ingestion key (safe to expose to the browser); host defaults to EU cloud. - posthog_key: str = "" - posthog_host: str = "https://eu.i.posthog.com" - # Intercom Messenger (support chat; treg's own workspace). Empty app_id = OFF, so self-hosted - # instances never load the widget. The app_id is public (visible in page source); the secret - # signs user_hash for identity verification and must never reach the browser. - intercom_app_id: str = "" - intercom_secret: str = "" - # Registry OAuth apps for the non-Google providers (oauth_providers.py). Empty = that provider - # is listed as unconfigured rather than failing part-way through a consent. - # treg's own Google Ads developer token, from OUR approved manager account. Ads needs it on - # every call ALONGSIDE the user's OAuth — it identifies the calling application, not the user, - # and grants no access to our ad accounts. Holding it centrally is the whole point: a user - # would otherwise wait weeks for Google to approve one of their own. - google_ads_developer_token: str = "" - # Google Ads consents through a DEDICATED OAuth client in its own Cloud project — NOT the shared - # google_client_id. A developer token is permanently welded to the first Cloud project it calls - # from, and the shared project is already paired to a different token, so Ads must use a client - # from the project its live token is paired with. Empty = Ads shows as unconfigured (correct — it - # can't work without this) rather than silently reusing the wrong client. - google_ads_client_id: str = "" - google_ads_client_secret: str = "" - # Google Ads conversion upload. This customer id and the refresh token below are both required; - # if either is empty the whole feature is OFF (tests stay inert and self-hosters send nothing) — - # the same gate shape analytics.py uses for posthog_key. - google_ads_customer_id: str = "" - # Manager (MCC) account used to access google_ads_customer_id. Optional for direct client auth; - # when present this is sent as login-customer-id. It cannot be inferred from Secret.resource_ref, - # which is the TARGET client account selected in discovery, not its manager. - google_ads_login_customer_id: str = "" - # treg's OWN long-lived refresh token for the Data Manager conversion uploader — a PLATFORM - # credential, obtained once, out of band, by an operator via the OAuth playground with scope - # https://www.googleapis.com/auth/datamanager. It is exchanged against `google_ads_client_id`/ - # `_secret` above (the refresh token is issued against that client, so it must be redeemed with - # it), never against a customer's own OAuth connection: the uploader ships treg's OWN marketing - # conversions to treg's OWN ad account, a different purpose from a customer connecting Ads to - # read their campaign data, and the two must not share a credential or a consent screen. Empty - # = the whole feature is OFF. See docs/context/architecture/ads-conversions.md. - ads_conv_refresh_token: str = "" - - linkedin_client_id: str = "" - linkedin_client_secret: str = "" - - slack_client_id: str = "" - slack_client_secret: str = "" - x_client_id: str = "" - x_client_secret: str = "" - # TikTok issues a separate app (and separate key/secret) per sandbox, so a dev deployment - # points at a sandbox client while production points at the reviewed one. - tiktok_client_id: str = "" - tiktok_client_secret: str = "" - # Facebook Login credentials. These continue to back Facebook Pages, Meta Ads, and the optional - # Page-based Instagram authorization method. - meta_client_id: str = "" - meta_client_secret: str = "" - # Business Login for Instagram has its own Instagram App ID and App Secret. Meta documents - # these under Instagram → API setup with Instagram login. They are not interchangeable with - # the Facebook Login app credentials above. - instagram_client_id: str = "" - instagram_client_secret: str = "" - # Advertising OAuth platforms — unset by default, so these providers list as "not configured" - # until this deployment registers its own developer app on each network. - microsoft_ads_client_id: str = "" - microsoft_ads_client_secret: str = "" - snapchat_ads_client_id: str = "" - snapchat_ads_client_secret: str = "" - tiktok_ads_client_id: str = "" - tiktok_ads_client_secret: str = "" - pinterest_client_id: str = "" - pinterest_client_secret: str = "" - - # Overridable for tests; real Google by default. - google_authorize_url: str = "https://accounts.google.com/o/oauth2/v2/auth" - google_token_url: str = "https://oauth2.googleapis.com/token" - google_userinfo_url: str = "https://openidconnect.googleapis.com/v1/userinfo" - - # Email one-time-code login (the third identity door). Dev mode RETURNS the code in the API - # response, which is an unauthenticated account-takeover vector in prod — so it defaults OFF and - # must be explicitly enabled (TREG_EMAIL_DEV_MODE=true) for local testing without a mail sender. - email_dev_mode: bool = False - - # Frictionless local mode: `curl … | sh` brings up a server you are already signed into, with no - # account, email or password. Only takes effect when `single_user_ok` allows it (see below). - single_user: bool = False - # Where the bootstrapped token is written so the CLI/installer can pick it up (0600). - single_user_token_file: str = "~/.treg/local-token" - - @property - def single_user_ok(self) -> bool: - """No-login mode: ONE person on ONE machine, so the dashboard opens already signed in. - - Guarded exactly like `expose_dev_code`, because a no-login dashboard reachable from the - internet would hand the whole registry to anyone who found it. Both must hold: - - a LOCAL sqlite database (a Postgres URL means a real deploy), and - - a loopback `public_url` (so it is not fronted by a public domain). - A stray TREG_SINGLE_USER=true in production therefore does nothing. - """ - if not self.single_user or "sqlite" not in self.database_url: - return False - host = (urlsplit(self.public_url).hostname or "").lower() - return host in ("localhost", "127.0.0.1", "0.0.0.0", "::1", "") - - @property - def platform_provider_set(self) -> frozenset[str]: - """The allow-listed tier-4 providers (comma-separated `TREG_PLATFORM_PROVIDERS`).""" - return frozenset(p.strip().lower() for p in self.platform_providers.split(",") if p.strip()) - - def platform_key_for(self, provider: str) -> str | None: - """treg's own key for `provider`, or None if tier 4 must not serve it. BOTH conditions have to - hold — the provider is allow-listed AND a key is configured — so neither half alone can start - spending our money. Returns the value only; callers put the SETTING NAME in the binding - (`platform_setting_name`) so the key itself never travels through a tool row.""" - if (provider or "").lower() not in self.platform_provider_set: - return None - return getattr(self, platform_setting_name(provider), "") or None - - @property - def platform_daily_cap_micro(self) -> int: - return int(round(self.platform_daily_cap_usd * 1_000_000)) - - @property - def overflow_daily_budget_micro(self) -> int: - return int(round(self.overflow_daily_budget_usd * 1_000_000)) - - def overflow_key_for(self, aggregator: str) -> str | None: - return getattr(self, f"overflow_key_{aggregator}", "") or None - - @property - def oauth_billed_set(self) -> frozenset[str]: - """OAuth providers whose registry-connect calls are metered (comma-separated - `TREG_OAUTH_BILLED_PROVIDERS`). Empty = the current free behavior.""" - return frozenset(p.strip().lower() for p in self.oauth_billed_providers.split(",") if p.strip()) - - @property - def expose_dev_code(self) -> bool: - """Dev login codes may be returned in the response ONLY on a local sqlite database — never on a - real (Postgres) deploy. So even a stray TREG_EMAIL_DEV_MODE=true in production can't leak codes.""" - return self.email_dev_mode and "sqlite" in self.database_url - - # Transactional email via Resend (OTP sign-in codes + team invitations). Empty key = no real - # send (dev mode still returns the code; prod without a key silently skips the send). From must - # be a Resend-verified domain — treg.to is verified (DKIM + SPF); treg.superdesign.dev remains - # verified as a fallback. - resend_api_key: str = "" - email_from: str = "tools-registry " - - -@lru_cache -def get_settings() -> Settings: - return Settings() +PLACEHOLDER_CONFIG \ No newline at end of file diff --git a/tests/test_key_providers.py b/tests/test_key_providers.py index ef2c7ec37..4b66d81a3 100644 --- a/tests/test_key_providers.py +++ b/tests/test_key_providers.py @@ -1,219 +1 @@ -"""API-key providers (auth_kind="key") — the marketplace's paste-a-key connect flow. - -These share Slack's bring-your-own-credential path (verify → store → auto-provision), differing only -in the header or query param the key rides in. The upstream is the shared in-process ASGI app from -conftest (`/whoami` echoes; `/units` and `/units-bad` model Semrush's plain-text balance responses). -""" - -from __future__ import annotations - -import dataclasses - -from httpx import AsyncClient - -from treg import oauth_providers as P - - -# ---- registry shape ---------------------------------------------------------------------- -def test_key_providers_are_offerable_without_deployment_credentials(): - """The user brings the key, so treg holds no app of its own — a key provider must be offerable, - not shown as 'not configured' the way an unset OAuth provider is.""" - for svc in ("apollo", "pdl", "akta", "hunter", "crunchbase", "tikhub", "brightdata", "semrush", - "justoneapi", "dataforseo", "seranking", "moz", "majestic", "serpstat", "exa", - "lusha", "coresignal", "diffbot", "thecompaniesapi", "leadmagic", "fiber-ai", - "companyenrich", "oceanio", "tomba", "predictleads", "findymail", "branddev", - "icypeas", "leadsforge", "influencersclub", "crustdata", "aviato", - "spyfu", "apify", "meta-ad-library", "serpapi", - "coingecko", "polygon", "finnhub", "twelvedata", "fmp", "eodhd", "marketstack", - "tiingo"): - p = P.get(svc) - assert p is not None, svc - assert p.auth_kind == "key", svc - assert p.uses_pasted_secret is True, svc - assert p.is_token_kind is False, f"{svc}: an API key is not a Slack bot token" - assert P.is_configured(p) is True, svc - - -def test_key_providers_appear_in_the_marketplace_listing(): - listing = {row["service"]: row for row in P.listing()} - assert listing["apollo"]["category"] == "Enrichment" - assert listing["apollo"]["auth_kind"] == "key" - assert listing["semrush"]["category"] == "SEO" - assert listing["tikhub"]["category"] == "Social media" - assert listing["coingecko"]["category"] == "Market data" - assert "Enrichment" in P.CATEGORY_ORDER - assert "Market data" in P.CATEGORY_ORDER - - -# ---- connect-by-key ---------------------------------------------------------------------- -async def test_key_connect_provisions_a_header_binding(clients: AsyncClient, monkeypatch): - """A header key (Apollo's X-Api-Key) is a plain string injected as an env header — never an - oauth blob with an access_token field that isn't there.""" - # token_verify_field cleared: the generic echo stub doesn't model Apollo's is_logged_in body; - # this test is about the binding shape, not Apollo's body check (covered separately below). - monkeypatch.setitem(P.REGISTRY, "apollo", dataclasses.replace( - P.REGISTRY["apollo"], base_url="http://upstream", probe_path="/whoami", token_verify_field="")) - r = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-apollo"}) - assert r.status_code == 200, r.text - assert r.json()["health"] == "ok", "a verified key is known-good, not 'unknown'" - - tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "apollo") - b = tool["bindings"][0] - assert b["injector"] == "env" and b["location"] == "header" - assert b["name"] == "X-Api-Key" and b["format"] == "{secret}" - assert "secret_field" not in b or b.get("secret_field") in (None, "") - - -async def test_required_provider_header_is_probed_bound_and_caller_proof(clients: AsyncClient, monkeypatch): - """A protocol header is provider metadata, not proxy behavior. It must be present during the - connect probe and become a constant binding that overwrites a caller's stale version.""" - monkeypatch.setitem(P.REGISTRY, "crustdata", dataclasses.replace( - P.REGISTRY["crustdata"], base_url="http://upstream", probe_path="/requires-version")) - r = await clients.post("/connections/token", json={"provider": "crustdata", "token": "cr-key"}) - assert r.status_code == 200, r.text - - tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "crustdata") - assert tool["bindings"] == [ - {"secret_id": tool["bindings"][0]["secret_id"], "injector": "env", "location": "header", - "name": "Authorization", "format": "Bearer {secret}"}, - {"secret_id": tool["bindings"][0]["secret_id"], "injector": "env", "location": "header", - "name": "x-api-version", "format": "2025-11-01"}, - ] - called = await clients.get( - "/call/crustdata/requires-version", headers={"x-api-version": "stale-version"}) - assert called.status_code == 200, called.text - assert called.json()["version"] == "2025-11-01" - - -async def test_key_connect_supports_a_query_param_key(clients: AsyncClient, monkeypatch): - """Semrush authenticates the classic API with ?key=… and answers the balance check in PLAIN - TEXT — the probe must not JSON-parse it, and the tool must bind the key as a query param.""" - monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( - P.REGISTRY["semrush"], base_url="http://upstream", probe_url="", probe_path="/units")) - r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-key"}) - assert r.status_code == 200, r.text - - tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "semrush") - b = tool["bindings"][0] - assert b["injector"] == "env" and b["location"] == "query" - assert b["name"] == "key" and b["format"] == "{secret}" - - -async def test_a_plain_text_error_body_is_rejected(clients: AsyncClient, monkeypatch): - """Semrush signals a bad key with HTTP 200 + an "ERROR ..." text body. Storing it anyway just - moves the failure to the first real report call, after the user has left the setup screen.""" - monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( - P.REGISTRY["semrush"], base_url="http://upstream", probe_url="", probe_path="/units-bad")) - r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-bad"}) - assert r.status_code == 422, r.text - assert "ERROR" in r.text - assert not [c for c in (await clients.get("/connections")).json() if c["provider"] == "semrush"] - - -async def test_a_200_with_a_false_verify_field_is_rejected(clients: AsyncClient, monkeypatch): - """Apollo answers HTTP 200 even for a bad key and signals validity in is_logged_in. token_verify_field - makes us read that field: a false one is rejected, a true one connects. Verified live against Apollo.""" - apollo = dataclasses.replace( - P.REGISTRY["apollo"], base_url="http://upstream", probe_path="/verify-field", - token_verify_field="is_logged_in") - monkeypatch.setitem(P.REGISTRY, "apollo", apollo) - - bad = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-bad"}) - assert bad.status_code == 422 and "is_logged_in" in bad.text - assert not [c for c in (await clients.get("/connections")).json() if c["provider"] == "apollo"] - - ok = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-good"}) - assert ok.status_code == 200, ok.text - - -async def test_probe_url_overrides_base_url_for_verification(clients: AsyncClient, monkeypatch): - """The cheapest key-check can live on a different host than the data API (Semrush's balance is on - www.semrush.com). probe_url must win: point it at a passing endpoint while probe_path would fail.""" - monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( - P.REGISTRY["semrush"], base_url="http://upstream", - probe_url="http://upstream/units", probe_path="/units-bad")) - r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-key"}) - assert r.status_code == 200, "probe_url must be used, not base_url + probe_path" - - -async def test_key_connection_lists_and_revokes(clients: AsyncClient, monkeypatch): - """A key connection is a real connection — it must be visible in the list and revocable by id.""" - monkeypatch.setitem(P.REGISTRY, "tikhub", dataclasses.replace( - P.REGISTRY["tikhub"], base_url="http://upstream", probe_path="/whoami")) - r = await clients.post("/connections/token", json={"provider": "tikhub", "token": "th-key"}) - assert r.status_code == 200, r.text - - listed = [c for c in (await clients.get("/connections")).json() if c["provider"] == "tikhub"] - assert len(listed) == 1 and listed[0]["kind"] == "env" - assert (await clients.delete(f"/connections/{listed[0]['id']}")).status_code == 200 - - -# ---- HTTP Basic providers: paste a raw pair OR a ready-made Base64 blob ---------------------- -async def test_basic_provider_encodes_a_raw_login_password_once(clients: AsyncClient, monkeypatch): - """DataForSEO/Moz take HTTP Basic. Pasting the raw `login:password`, treg Base64s it once and the - upstream sees `Basic `.""" - import base64 - monkeypatch.setitem(P.REGISTRY, "dataforseo", dataclasses.replace( - P.REGISTRY["dataforseo"], base_url="http://upstream", probe_path="/whoami")) - r = await clients.post("/connections/token", json={"provider": "dataforseo", "token": "login:pw"}) - assert r.status_code == 200, r.text - echoed = (await clients.get("/call/dataforseo/whoami")).json()["auth"] - assert echoed == "Basic " + base64.b64encode(b"login:pw").decode() - - -async def test_basic_provider_accepts_a_ready_made_base64_blob(clients: AsyncClient, monkeypatch): - """The DataForSEO and Moz dashboards ALSO hand out a ready-made Base64 credential, and users paste - that at least as often as the raw pair. treg must NOT Base64 it a second time — the upstream has to - receive exactly `Basic `, decoding back to the original `login:password`.""" - import base64 - blob = base64.b64encode(b"login:pw").decode() - monkeypatch.setitem(P.REGISTRY, "dataforseo", dataclasses.replace( - P.REGISTRY["dataforseo"], base_url="http://upstream", probe_path="/whoami")) - r = await clients.post("/connections/token", json={"provider": "dataforseo", "token": blob}) - assert r.status_code == 200, r.text - echoed = (await clients.get("/call/dataforseo/whoami")).json()["auth"] - assert echoed == "Basic " + blob, "a pasted Base64 blob must not be double-encoded" - - -# ---- corrected probe shapes (regression guards for the 2026-08-13 connect-flow fixes) -------- -def test_brightdata_probe_is_a_real_route(): - """The old /datasets/v3/datasets 404'd even for a valid token, refusing every real key. /status is - the free account check that answers 200 (valid) / 401 (bad).""" - assert P.get("brightdata").probe_path == "/status" - - -def test_justoneapi_probe_uses_camelcase_uniqueid(): - """snake_case unique_id made the API answer HTTP 400 ('must input one of them (uniqueId or - secUid)') and a VALID token was refused. The param is camelCase.""" - assert "uniqueId=" in P.get("justoneapi").probe_path - assert "unique_id=" not in P.get("justoneapi").probe_path - - -async def test_probe_parses_json_body_labelled_text_plain(clients: AsyncClient, monkeypatch): - """ScrapeCreators answers HTTP 200 with a JSON body but a text/plain content-type; validity lives - in creditCount. Gating the parse on application/json left the field unread and refused a good key.""" - monkeypatch.setitem(P.REGISTRY, "scrapecreators", dataclasses.replace( - P.REGISTRY["scrapecreators"], base_url="http://upstream", - probe_path="/credit-json-as-text", token_verify_field="creditCount")) - r = await clients.post("/connections/token", json={"provider": "scrapecreators", "token": "sc-key"}) - assert r.status_code == 200, r.text - - -async def test_probe_keeps_a_query_string_baked_into_probe_path(clients: AsyncClient, monkeypatch): - """A probe_path like `/autocomplete?field=title` must reach the upstream WITH that query — httpx - drops a URL's own query when params= is passed, which used to 400 the probe and refuse the key.""" - monkeypatch.setitem(P.REGISTRY, "pdl", dataclasses.replace( - P.REGISTRY["pdl"], base_url="http://upstream", - probe_path="/needs-query?field=title&text=data", token_verify_field="")) - r = await clients.post("/connections/token", json={"provider": "pdl", "token": "pdl-key"}) - assert r.status_code == 200, r.text - - -async def test_query_token_survives_alongside_a_probe_path_query(clients: AsyncClient, monkeypatch): - """A query-credential provider (SpyFu: ?api_key=) whose probe_path ALSO carries a required query - (?domain=) must send both — the merge keeps the path's params and adds the credential on top.""" - monkeypatch.setitem(P.REGISTRY, "spyfu", dataclasses.replace( - P.REGISTRY["spyfu"], base_url="http://upstream", - probe_path="/needs-query?field=title", token_verify_field="")) - r = await clients.post("/connections/token", json={"provider": "spyfu", "token": "spyfu-secret"}) - assert r.status_code == 200, r.text +PLACEHOLDER_TEST_KEY \ No newline at end of file diff --git a/tests/test_oauth_providers_m3.py b/tests/test_oauth_providers_m3.py index 7fd7cbeba..4e7983307 100644 --- a/tests/test_oauth_providers_m3.py +++ b/tests/test_oauth_providers_m3.py @@ -1,367 +1 @@ -"""Milestone 3 — the rest of the Google family, Slack, and X. - -X is the interesting one: it rejects an authorization code exchanged without a PKCE verifier, and -rejects the client secret in the request body. Both quirks are captured on the pending connect at -start time so the callback exchanges the code exactly the way the consent URL was built. -""" - -from __future__ import annotations - -import json -from dataclasses import replace -from urllib.parse import parse_qs, urlsplit - -import pytest -from httpx import AsyncClient -from sqlmodel import select - -from treg import oauth -from treg import oauth_providers as P -from treg.config import get_settings -from treg.infra.db import session_maker -from treg.models import PendingOAuth - - -@pytest.fixture -def all_apps(monkeypatch): - for k in ("GOOGLE", "SLACK", "X", "TIKTOK"): - monkeypatch.setenv(f"TREG_{k}_CLIENT_ID", f"{k.lower()}-cid") - monkeypatch.setenv(f"TREG_{k}_CLIENT_SECRET", f"{k.lower()}-csec") - monkeypatch.setenv("TREG_META_CLIENT_ID", "meta-cid") - monkeypatch.setenv("TREG_META_CLIENT_SECRET", "meta-csec") - monkeypatch.setenv("TREG_INSTAGRAM_CLIENT_ID", "instagram-cid") - monkeypatch.setenv("TREG_INSTAGRAM_CLIENT_SECRET", "instagram-csec") - get_settings.cache_clear() - yield - get_settings.cache_clear() - - -def _q(payload: dict) -> dict: - return parse_qs(urlsplit(payload["consent_url"]).query) - - -# ---- registry shape ---------------------------------------------------------------------- -def test_every_provider_is_registered(): - assert set(P.REGISTRY) == { - "google-search-console", "google-analytics", "google-business-profile", "google-tag-manager", - "google-ads", "youtube", "linkedin", "slack", "x", "tiktok", - "facebook", "instagram", "meta-ads", - # API-key providers (auth_kind="key") - "apollo", "pdl", "akta", "hunter", "crunchbase", "tikhub", "brightdata", "semrush", "justoneapi", - "scrapecreators", - "dataforseo", "seranking", "moz", "majestic", "serpstat", "exa", - "lusha", "coresignal", "diffbot", "thecompaniesapi", "leadmagic", "fiber-ai", - "companyenrich", "oceanio", "tomba", "predictleads", "findymail", "branddev", - "icypeas", "leadsforge", "influencersclub", "crustdata", "aviato", - "spyfu", "apify", "meta-ad-library", "serpapi", - "coingecko", "polygon", "finnhub", "twelvedata", "fmp", "eodhd", "marketstack", "tiingo", - "microsoft-ads", "snapchat-ads", "tiktok-ads", "pinterest-ads", - } - - -def test_default_capability_is_the_broadest(): - """Connect asks for the fullest capability; a narrower one is chosen up front, not bolted on - afterwards. Every provider's write must be a superset of its read for that to be safe.""" - assert P.GOOGLE_SEARCH_CONSOLE.default_capability == "write" - assert P.X.default_capability == "write" - assert P.GOOGLE_ADS.default_capability == "manage" # it has no read-only mode - assert P.GOOGLE_TAG_MANAGER.default_capability == "manage" - for provider in P.REGISTRY.values(): - caps = provider.capabilities - if "read" in caps and "write" in caps: - assert set(provider.scopes["read"]) < set(provider.scopes["write"]), provider.service - - -def test_google_ads_refuses_to_autoprovision(): - """Ads needs a developer-token header too; a bearer-only tool would 401 on first use.""" - assert P.GOOGLE_ADS.can_autoprovision is False - assert "developer-token" in P.GOOGLE_ADS.extra_credential_note - assert P.GOOGLE_SEARCH_CONSOLE.can_autoprovision is True - - -def test_x_write_keeps_offline_access(): - """Without offline.access the token can't be refreshed and every X connection becomes a - manual-reconnect chore within hours.""" - assert "offline.access" in P.X.scopes_for("write") - assert "offline.access" in P.X.scopes_for("read") - - -# ---- X's two quirks ---------------------------------------------------------------------- -async def test_x_consent_url_carries_a_pkce_challenge(clients: AsyncClient, all_apps): - d = (await clients.post("/oauth/start", json={"provider": "x"})).json() - q = _q(d) - assert q["code_challenge_method"] == ["S256"] - assert q["code_challenge"], "X rejects a code exchanged without a verifier" - # the verifier itself must stay server-side - assert "code_verifier" not in q - - -async def test_pkce_challenge_matches_the_stored_verifier(clients: AsyncClient, all_apps): - d = (await clients.post("/oauth/start", json={"provider": "x"})).json() - challenge = _q(d)["code_challenge"][0] - async with session_maker() as db: - p = (await db.execute(select(PendingOAuth).where(PendingOAuth.state == d["state"]))).scalars().one() - assert p.code_verifier - assert oauth.pkce_challenge(p.code_verifier) == challenge - assert p.token_endpoint_auth_method == "client_secret_basic" - - -async def test_google_does_not_use_pkce(clients: AsyncClient, all_apps): - d = (await clients.post("/oauth/start", json={"provider": "google-search-console"})).json() - assert "code_challenge" not in _q(d) - - -# ---- TikTok's two quirks ------------------------------------------------------------------- -async def test_tiktok_consent_url_uses_client_key_not_client_id(clients: AsyncClient, all_apps): - """TikTok ignores the OAuth2 spelling. Sending `client_id` gets a consent page that errors out - rather than an obvious 400, so this is worth pinning.""" - q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) - assert q["client_key"] == ["tiktok-cid"] - assert "client_id" not in q - - -async def test_tiktok_comma_joins_its_scopes(clients: AsyncClient, all_apps): - """Space-joined scopes come back from TikTok as scope_not_authorized — it splits on commas.""" - q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) - scope = q["scope"][0] - assert "," in scope and " " not in scope - assert set(scope.split(",")) == set(P.TIKTOK.scopes_for(P.TIKTOK.default_capability)) - - -async def test_tiktok_granted_scopes_are_stored_space_joined(clients: AsyncClient, all_apps, monkeypatch): - """The wire dialect must not leak into storage: every reader of granted_scopes uses .split(), - so a comma-joined grant would read as one bogus scope and report every capability unsatisfied.""" - # Registry mode takes token_uri from the provider, not the body, so point the provider itself at - # the in-process upstream (frozen dataclass → replace rather than setattr). - monkeypatch.setitem(P.REGISTRY, "tiktok", replace(P.TIKTOK, token_uri="http://upstream/token")) - body = {"provider": "tiktok", "capability": "post"} - state = (await clients.post("/oauth/start", json=body)).json()["state"] - await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") - sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] - - conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] - assert set(conn["capabilities"]) == {"read", "draft", "post"} - - -def test_tiktok_capabilities_are_cumulative(): - """draft must contain read and post must contain draft, or satisfied_capabilities() (which is - set-containment) reports a connection that can post but cannot read.""" - t = P.TIKTOK - assert set(t.scopes_for("read")) < set(t.scopes_for("draft")) < set(t.scopes_for("post")) - assert t.default_capability == "post" - # video.publish is the whole difference between "we drafted it for you" and "we posted it". - assert "video.publish" not in t.scopes_for("draft") - assert "video.publish" in t.scopes_for("post") - - -# ---- per-provider consent params --------------------------------------------------------- -async def test_google_keeps_offline_consent_params(clients: AsyncClient, all_apps): - """access_type=offline + prompt=consent is what guarantees Google returns a refresh_token.""" - q = _q((await clients.post("/oauth/start", json={"provider": "google-search-console"})).json()) - assert q["access_type"] == ["offline"] and q["prompt"] == ["consent"] - - -def test_slack_is_bring_your_own_bot(): - """A Slack bot is workspace-scoped and belongs to the workspace it's installed in. A shared - treg app would sit between a team and their own messages — and couldn't be installed on their - behalf anyway — so the user brings their own token instead of consenting to ours.""" - assert P.SLACK.auth_kind == "token" - assert P.SLACK.scopes == {}, "no consent screen means no capability sizing" - assert P.SLACK.default_capability == "", "and nothing to default to" - - -async def test_each_provider_uses_its_own_client_credentials(clients: AsyncClient, all_apps): - for service, expected in (("google-search-console", "google-cid"), - ("google-tag-manager", "google-cid"), ("x", "x-cid")): - q = _q((await clients.post("/oauth/start", json={"provider": service})).json()) - assert q["client_id"] == [expected], service - - -# ---- scope gap detection (the re-consent trigger) ----------------------------------------- -def test_satisfied_capabilities_detects_a_scope_gap(): - """Providers never backfill scopes onto an issued grant — a later capability needs re-consent, - and this is how we know to prompt instead of letting the call 403.""" - gsc = P.GOOGLE_SEARCH_CONSOLE - read_only = gsc.scopes_for("read") - assert gsc.satisfied_capabilities(read_only) == ["read"] - assert "write" not in gsc.satisfied_capabilities(read_only) - both = read_only + gsc.scopes_for("write") - assert set(gsc.satisfied_capabilities(both)) == {"read", "write"} - - -def test_google_tag_manager_capabilities_are_cumulative_and_exclude_admin(): - """GTM can audit, prepare and publish without authority to delete an entire container or - administer the account's users. Each wider tier must still satisfy every narrower tier.""" - gtm = P.GOOGLE_TAG_MANAGER - assert set(gtm.scopes_for("read")) < set(gtm.scopes_for("write")) < set(gtm.scopes_for("manage")) - assert gtm.default_capability == "manage" - requested = {scope for scopes in gtm.scopes.values() for scope in scopes} - assert not { - "https://www.googleapis.com/auth/tagmanager.delete.containers", - "https://www.googleapis.com/auth/tagmanager.manage.users", - "https://www.googleapis.com/auth/tagmanager.manage.accounts", - } & requested - assert gtm.probe_path == gtm.discover_path == "/tagmanager/v2/accounts" - assert gtm.discover_key == "account" - assert gtm.discover_id_field == "path" - assert gtm.discover_label_field == "name" - - -async def test_unconfigured_providers_are_listed_but_flagged(clients: AsyncClient, monkeypatch): - monkeypatch.setenv("TREG_X_CLIENT_ID", "") - monkeypatch.setenv("TREG_X_CLIENT_SECRET", "") - get_settings.cache_clear() - try: - rows = {p["service"]: p for p in (await clients.get("/oauth/providers")).json()} - assert rows["x"]["configured"] is False - r = await clients.post("/oauth/start", json={"provider": "x"}) - assert r.status_code == 422 and "not configured" in r.text - # a bring-your-own-token provider needs nothing from the deployment, so it stays offerable - assert rows["slack"]["configured"] is True - finally: - get_settings.cache_clear() - - -# ---- the gap, surfaced on the connection -------------------------------------------------- -async def test_connection_reports_the_capability_it_lacks(clients: AsyncClient, all_apps): - """Connect read-only, then see that `write` is named as missing — the reconnect trigger.""" - body = { - "provider": "google-search-console", "capability": "read", - "token_uri": "http://upstream/token", # the in-process upstream stands in for Google - } - state = (await clients.post("/oauth/start", json=body)).json()["state"] - await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") - sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] - - conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] - assert conn["capabilities"] == ["read"] - assert conn["missing_capabilities"] == ["write"] - - -async def test_byo_connection_has_no_capability_fields(clients: AsyncClient): - body = {"name": "byo", "client_id": "c", "client_secret": "s", - "auth_uri": "http://p/auth", "token_uri": "http://upstream/token", "scopes": ["x"]} - state = (await clients.post("/oauth/start", json=body)).json()["state"] - await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") - sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] - conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] - assert "missing_capabilities" not in conn # nothing to compare against without a provider - - -# ---- LinkedIn ----------------------------------------------------------------------------- -def test_linkedin_has_one_capability(): - """These scopes let a member read their own profile and post as themselves. A read-only - LinkedIn connection could do nothing but identify you, so there is no second option worth - asking about — and a dialog with one real choice is just friction.""" - assert P.LINKEDIN.capabilities == ["write"] - assert "w_member_social" in P.LINKEDIN.scopes_for("write") - - -async def test_linkedin_does_not_get_googles_consent_params(clients: AsyncClient, monkeypatch): - monkeypatch.setenv("TREG_LINKEDIN_CLIENT_ID", "li-cid") - monkeypatch.setenv("TREG_LINKEDIN_CLIENT_SECRET", "li-csec") - get_settings.cache_clear() - try: - d = (await clients.post("/oauth/start", json={"provider": "linkedin"})).json() - q = _q(d) - assert q["client_id"] == ["li-cid"] - assert "access_type" not in q and "prompt" not in q, "LinkedIn rejects Google's params" - assert "w_member_social" in q["scope"][0] - finally: - get_settings.cache_clear() - - -def test_instagram_direct_and_page_grants_use_explicit_app_profiles(): - assert P.INSTAGRAM.client_id_setting == "instagram_client_id" - parsed = urlsplit(P.INSTAGRAM.base_url) - assert parsed.scheme == "https" - assert parsed.hostname == "graph.instagram.com" - page = P.INSTAGRAM.profile_for_authorization("facebook-page") - assert page.client_id_setting == P.FACEBOOK.client_id_setting == "meta_client_id" - assert page.base_url == P.FACEBOOK.base_url - - -def test_meta_capabilities_are_cumulative(): - """satisfied_capabilities() is set containment, so a non-cumulative tier would report a - connection that can publish but 'cannot read' — and the default capability would be wrong. - default_capability is the BROADEST tier by design (one honest consent screen beats - connect-twice), so adding manage moved the default there.""" - for provider in (P.FACEBOOK, P.INSTAGRAM): - assert set(provider.scopes["read"]) < set(provider.scopes["post"]), provider.service - assert set(provider.scopes["post"]) < set(provider.scopes["manage"]), provider.service - assert provider.default_capability == "manage", provider.service - - -def test_meta_messaging_stays_out_of_the_publish_tier(): - """A publish-only connect must never put "manage your messages" (or lead retrieval, or the - Page's Messenger inbox) on the consent screen — the two-way surfaces live only in manage.""" - two_way = { - "instagram_manage_messages", "instagram_manage_comments", "pages_messaging", - "pages_manage_engagement", "leads_retrieval", "catalog_management", - } - for provider in (P.FACEBOOK, P.INSTAGRAM): - for cap in ("read", "post"): - assert not two_way & set(provider.scopes[cap]), (provider.service, cap) - assert "instagram_business_manage_messages" in P.INSTAGRAM.scopes["manage"] - assert {"instagram_manage_messages", "pages_messaging"} <= set(P.INSTAGRAM.scopes["page-tools"]) - - -def test_lead_retrieval_brings_its_required_rider(): - """Meta only honors leads_retrieval alongside pages_manage_ads — requesting one without the - other consents fine and then 400s on /leads, which would demo as a broken integration.""" - manage = set(P.FACEBOOK.scopes["manage"]) - assert {"leads_retrieval", "pages_manage_ads"} <= manage - - -def test_instagram_login_is_direct_and_page_discovery_is_optional(): - for cap in ("read", "post", "manage"): - assert "pages_show_list" not in P.INSTAGRAM.scopes[cap] - assert P.INSTAGRAM.identity_required is True - page = P.INSTAGRAM.profile_for_authorization("facebook-page") - assert page.discover_id_field == "instagram_business_account.id" - assert "pages_show_list" in P.INSTAGRAM.scopes["page-tools"] - - -def test_meta_asks_for_a_long_lived_token(): - """Meta's code exchange yields a ~1-2h token and no refresh_token. Without the second - exchange every Meta connection dies the day it is made.""" - assert P.FACEBOOK.long_lived_exchange - assert P.INSTAGRAM.long_lived_exchange_style == "instagram" - assert not P.TIKTOK.long_lived_exchange # nothing else should have picked it up - - -def test_instagram_consent_never_mentions_page_publishing(): - """Scopes are per capability. An Instagram connect asking for pages_manage_posts would put - 'manage your Pages' posts' on the consent screen for authority it never uses.""" - for cap in P.INSTAGRAM.scopes.values(): - assert "pages_manage_posts" not in cap - - -def test_meta_page_discovery_can_walk_the_business_graph(): - """Most agency-held Pages (and the Instagram accounts linked to them) are OWNED by a Business - portfolio, where the member has business-level access and no personal Page role. Drop - business_management from either capability and that user consents cleanly, then gets an empty - picker — the extra listing 400s and is (rightly) swallowed.""" - for cap in P.FACEBOOK.scopes.values(): - assert "business_management" in cap - page = P.INSTAGRAM.profile_for_authorization("facebook-page") - assert "business_management" in P.INSTAGRAM.scopes["page-tools"] - for provider in (P.FACEBOOK, page): - assert provider.discover_extra_path.startswith("/me/businesses"), provider.service - assert provider.discover_extra_list_paths, provider.service - - -def test_meta_ads_needs_no_second_credential(): - """Google Ads is gated on a developer token from an approved manager account; Meta has no - equivalent, so a Meta Ads connect must yield a callable tool on its own.""" - assert P.META_ADS.can_autoprovision is True - assert P.META_ADS.needs_extra_credential is False - - -def test_meta_ads_read_can_still_list_accounts(): - """/me/adaccounts is a Business asset listing. Drop business_management from read and the - connect consents cleanly, then offers an empty account picker.""" - for cap in P.META_ADS.scopes.values(): - assert "business_management" in cap - assert set(P.META_ADS.scopes["read"]) < set(P.META_ADS.scopes["manage"]) - assert "ads_management" not in P.META_ADS.scopes["read"], "read must not be able to spend money" +PLACEHOLDER_TEST_OAUTH \ No newline at end of file From 134c58264092a43422102856bc2bec3defe3d10b Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 14:36:22 +0200 Subject: [PATCH 03/31] Register Feedjolt as a BYOK catalog provider. --- src/treg/catalog/feedjolt.yaml | 438 ++++++++++++++++++++++++++++++++- 1 file changed, 437 insertions(+), 1 deletion(-) diff --git a/src/treg/catalog/feedjolt.yaml b/src/treg/catalog/feedjolt.yaml index 3b9f9c796..d9e8d674b 100644 --- a/src/treg/catalog/feedjolt.yaml +++ b/src/treg/catalog/feedjolt.yaml @@ -1 +1,437 @@ -PLACEHOLDER_YAML \ No newline at end of file +provider: feedjolt +source: + docs: https://www.feedjolt.com/en/docs/developers + openapi: https://api.feedjolt.com/openapi.json + curated: 2026-09-02 +# BYOK listing. REST+MCP sit in Feedjolt's premium set (Startup $9/mo monthly, Scale $15/mo +# annual). There is no per-call meter and no rate-card endpoint, so treg cannot serve these on +# its own key yet. Authenticated routes are own_account workspace data. +# +# MCP (Streamable HTTP at /mcp/reader and /mcp/writer) is a sibling product, not catalogued here — +# treg relays REST HTTP, not JSON-RPC. +# +# Probe is GET /workspaces. GET /health is unauthenticated (200 with no key) and must not be used. + +limits: "plan-included; 429 on abuse. REST API is not on the Growth plan." +pricing_url: https://www.feedjolt.com/en + +proposed_capabilities: + product.feedback.boards.list: "List feedback boards in a product workspace" + product.feedback.posts.list: "List customer feedback posts" + product.feedback.posts.get: "Get one feedback post" + product.feedback.posts.search: "Search customer feedback posts by keyword" + product.feedback.roadmap.get: "Get a public product roadmap grouped by status" + product.feedback.changelog.list: "List published changelog entries" + product.feedback.statuses.list: "List feedback statuses in a workspace" + product.feedback.tags.list: "List tags in a feedback workspace" + +endpoints: + - id: feedjolt.public.boards.list + capability: product.feedback.boards.list + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/boards + name: "List public boards" + summary: "List Public Boards" + input: + pathParams: + workspace_slug: {type: string, required: true, note: "workspace slug", example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read; no API key required. Included in plan when a key is used." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.public.posts.list + capability: product.feedback.posts.list + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/boards/{board_slug}/posts + name: "List public posts on a board" + summary: "List Public Posts" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + board_slug: {type: string, required: true, example: "feedback"} + queryParams: + page: {type: integer, required: false, example: 1} + page_size: {type: integer, required: false, note: "price dial; keep small", example: 3} + sort_by: {type: string, required: false, note: "default newest"} + q: {type: string, required: false} + status_id: {type: string, required: false} + tag_ids: {type: string, required: false} + test_request: + pathParams: {workspace_slug: "corvidly", board_slug: "feedback"} + queryParams: {page_size: 3} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read; no API key required." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.public.posts.get + capability: product.feedback.posts.get + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/posts/{post_id} + name: "Get a public post" + summary: "Get Public Post" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + post_id: {type: string, required: true, example: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + test_request: + pathParams: {workspace_slug: "corvidly", post_id: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read; no API key required. Test id harvested from the board list on 2026-09-02." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.public.posts.search + capability: product.feedback.posts.search + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/search + name: "Search public posts" + summary: "Search Public Posts" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + q: {type: string, required: true, example: "MCP"} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {q: "MCP"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read. Query 'login' returned 0 hits; 'MCP' hits real posts." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.public.roadmap.get + capability: product.feedback.roadmap.get + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/roadmap + name: "Get public roadmap" + summary: "Get Public Roadmap" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + tags: {type: string, required: false, note: "comma-separated tag ids"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read. corvidly returned empty buckets on 2026-09-02; still a 200." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.public.changelog.list + capability: product.feedback.changelog.list + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/changelog + name: "List public changelog" + summary: "List Public Changelog" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + page: {type: integer, required: false, example: 1} + page_size: {type: integer, required: false, example: 5} + version_id: {type: string, required: false} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {page_size: 5} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read. corvidly returned an empty list on 2026-09-02; still a 200." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.account.workspaces.list + kind: account + capability: account.usage + platform: account + scope: own_account + method: GET + path: /workspaces + name: "List workspaces for this API key" + summary: "List Workspaces" + input: + note: "no parameters — the key rides in the Authorization Bearer header. This is the registry probe." + test_request: {queryParams: {}} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Probe. Garbage key returns 401 {\"detail\":\"Not authenticated\"}. Valid-key 2xx pending maintainer credential." + docs_url: https://www.feedjolt.com/en/docs/developers + + - id: feedjolt.boards.list + capability: product.feedback.boards.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/boards + name: "List boards in a workspace" + summary: "List Boards" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.posts.list + capability: product.feedback.posts.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/posts + name: "List posts in a workspace" + summary: "List Posts" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + page: {type: integer, required: false, example: 1} + page_size: {type: integer, required: false, note: "keep small", example: 5} + board_id: {type: string, required: false} + status_id: {type: string, required: false} + q: {type: string, required: false} + sort_by: {type: string, required: false, note: "default newest"} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {page_size: 5} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.posts.search + capability: product.feedback.posts.search + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/posts/search + name: "Search posts in a workspace" + summary: "Search Posts" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + q: {type: string, required: true, example: "MCP"} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {q: "MCP"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.posts.get + capability: product.feedback.posts.get + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/posts/{post_id} + name: "Get a post" + summary: "Get a post in a workspace" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + post_id: {type: string, required: true, example: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + test_request: + pathParams: {workspace_slug: "corvidly", post_id: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.roadmap.get + capability: product.feedback.roadmap.get + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/roadmap + name: "Get workspace roadmap" + summary: "Get Public Roadmap" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.changelog.list + capability: product.feedback.changelog.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/changelog + name: "List workspace changelog" + summary: "List Public Changelog" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + page: {type: integer, required: false, example: 1} + page_size: {type: integer, required: false, example: 5} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {page_size: 5} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.statuses.list + capability: product.feedback.statuses.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/statuses + name: "List statuses" + summary: "List Statuses" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.tags.list + capability: product.feedback.tags.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/tags + name: "List tags" + summary: "List Tags" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs From a847be1a4a1c2594255ca7a3147c2b022584b0f4 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 14:38:10 +0200 Subject: [PATCH 04/31] Register Feedjolt as a BYOK catalog provider. --- tests/test_key_providers.py | 220 +++++++++++++++++- tests/test_oauth_providers_m3.py | 368 ++++++++++++++++++++++++++++++- 2 files changed, 586 insertions(+), 2 deletions(-) diff --git a/tests/test_key_providers.py b/tests/test_key_providers.py index 4b66d81a3..86b3c8ab0 100644 --- a/tests/test_key_providers.py +++ b/tests/test_key_providers.py @@ -1 +1,219 @@ -PLACEHOLDER_TEST_KEY \ No newline at end of file +"""API-key providers (auth_kind="key") — the marketplace's paste-a-key connect flow. + +These share Slack's bring-your-own-credential path (verify → store → auto-provision), differing only +in the header or query param the key rides in. The upstream is the shared in-process ASGI app from +conftest (`/whoami` echoes; `/units` and `/units-bad` model Semrush's plain-text balance responses). +""" + +from __future__ import annotations + +import dataclasses + +from httpx import AsyncClient + +from treg import oauth_providers as P + + +# ---- registry shape ---------------------------------------------------------------------- +def test_key_providers_are_offerable_without_deployment_credentials(): + """The user brings the key, so treg holds no app of its own — a key provider must be offerable, + not shown as 'not configured' the way an unset OAuth provider is.""" + for svc in ("apollo", "pdl", "akta", "hunter", "feedjolt", "crunchbase", "tikhub", "brightdata", "semrush", + "justoneapi", "dataforseo", "seranking", "moz", "majestic", "serpstat", "exa", + "lusha", "coresignal", "diffbot", "thecompaniesapi", "leadmagic", "fiber-ai", + "companyenrich", "oceanio", "tomba", "predictleads", "findymail", "branddev", + "icypeas", "leadsforge", "influencersclub", "crustdata", "aviato", + "spyfu", "apify", "meta-ad-library", "serpapi", + "coingecko", "polygon", "finnhub", "twelvedata", "fmp", "eodhd", "marketstack", + "tiingo"): + p = P.get(svc) + assert p is not None, svc + assert p.auth_kind == "key", svc + assert p.uses_pasted_secret is True, svc + assert p.is_token_kind is False, f"{svc}: an API key is not a Slack bot token" + assert P.is_configured(p) is True, svc + + +def test_key_providers_appear_in_the_marketplace_listing(): + listing = {row["service"]: row for row in P.listing()} + assert listing["apollo"]["category"] == "Enrichment" + assert listing["apollo"]["auth_kind"] == "key" + assert listing["semrush"]["category"] == "SEO" + assert listing["tikhub"]["category"] == "Social media" + assert listing["coingecko"]["category"] == "Market data" + assert "Enrichment" in P.CATEGORY_ORDER + assert "Market data" in P.CATEGORY_ORDER + + +# ---- connect-by-key ---------------------------------------------------------------------- +async def test_key_connect_provisions_a_header_binding(clients: AsyncClient, monkeypatch): + """A header key (Apollo's X-Api-Key) is a plain string injected as an env header — never an + oauth blob with an access_token field that isn't there.""" + # token_verify_field cleared: the generic echo stub doesn't model Apollo's is_logged_in body; + # this test is about the binding shape, not Apollo's body check (covered separately below). + monkeypatch.setitem(P.REGISTRY, "apollo", dataclasses.replace( + P.REGISTRY["apollo"], base_url="http://upstream", probe_path="/whoami", token_verify_field="")) + r = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-apollo"}) + assert r.status_code == 200, r.text + assert r.json()["health"] == "ok", "a verified key is known-good, not 'unknown'" + + tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "apollo") + b = tool["bindings"][0] + assert b["injector"] == "env" and b["location"] == "header" + assert b["name"] == "X-Api-Key" and b["format"] == "{secret}" + assert "secret_field" not in b or b.get("secret_field") in (None, "") + + +async def test_required_provider_header_is_probed_bound_and_caller_proof(clients: AsyncClient, monkeypatch): + """A protocol header is provider metadata, not proxy behavior. It must be present during the + connect probe and become a constant binding that overwrites a caller's stale version.""" + monkeypatch.setitem(P.REGISTRY, "crustdata", dataclasses.replace( + P.REGISTRY["crustdata"], base_url="http://upstream", probe_path="/requires-version")) + r = await clients.post("/connections/token", json={"provider": "crustdata", "token": "cr-key"}) + assert r.status_code == 200, r.text + + tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "crustdata") + assert tool["bindings"] == [ + {"secret_id": tool["bindings"][0]["secret_id"], "injector": "env", "location": "header", + "name": "Authorization", "format": "Bearer {secret}"}, + {"secret_id": tool["bindings"][0]["secret_id"], "injector": "env", "location": "header", + "name": "x-api-version", "format": "2025-11-01"}, + ] + called = await clients.get( + "/call/crustdata/requires-version", headers={"x-api-version": "stale-version"}) + assert called.status_code == 200, called.text + assert called.json()["version"] == "2025-11-01" + + +async def test_key_connect_supports_a_query_param_key(clients: AsyncClient, monkeypatch): + """Semrush authenticates the classic API with ?key=… and answers the balance check in PLAIN + TEXT — the probe must not JSON-parse it, and the tool must bind the key as a query param.""" + monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( + P.REGISTRY["semrush"], base_url="http://upstream", probe_url="", probe_path="/units")) + r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-key"}) + assert r.status_code == 200, r.text + + tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "semrush") + b = tool["bindings"][0] + assert b["injector"] == "env" and b["location"] == "query" + assert b["name"] == "key" and b["format"] == "{secret}" + + +async def test_a_plain_text_error_body_is_rejected(clients: AsyncClient, monkeypatch): + """Semrush signals a bad key with HTTP 200 + an "ERROR ..." text body. Storing it anyway just + moves the failure to the first real report call, after the user has left the setup screen.""" + monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( + P.REGISTRY["semrush"], base_url="http://upstream", probe_url="", probe_path="/units-bad")) + r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-bad"}) + assert r.status_code == 422, r.text + assert "ERROR" in r.text + assert not [c for c in (await clients.get("/connections")).json() if c["provider"] == "semrush"] + + +async def test_a_200_with_a_false_verify_field_is_rejected(clients: AsyncClient, monkeypatch): + """Apollo answers HTTP 200 even for a bad key and signals validity in is_logged_in. token_verify_field + makes us read that field: a false one is rejected, a true one connects. Verified live against Apollo.""" + apollo = dataclasses.replace( + P.REGISTRY["apollo"], base_url="http://upstream", probe_path="/verify-field", + token_verify_field="is_logged_in") + monkeypatch.setitem(P.REGISTRY, "apollo", apollo) + + bad = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-bad"}) + assert bad.status_code == 422 and "is_logged_in" in bad.text + assert not [c for c in (await clients.get("/connections")).json() if c["provider"] == "apollo"] + + ok = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-good"}) + assert ok.status_code == 200, ok.text + + +async def test_probe_url_overrides_base_url_for_verification(clients: AsyncClient, monkeypatch): + """The cheapest key-check can live on a different host than the data API (Semrush's balance is on + www.semrush.com). probe_url must win: point it at a passing endpoint while probe_path would fail.""" + monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( + P.REGISTRY["semrush"], base_url="http://upstream", + probe_url="http://upstream/units", probe_path="/units-bad")) + r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-key"}) + assert r.status_code == 200, "probe_url must be used, not base_url + probe_path" + + +async def test_key_connection_lists_and_revokes(clients: AsyncClient, monkeypatch): + """A key connection is a real connection — it must be visible in the list and revocable by id.""" + monkeypatch.setitem(P.REGISTRY, "tikhub", dataclasses.replace( + P.REGISTRY["tikhub"], base_url="http://upstream", probe_path="/whoami")) + r = await clients.post("/connections/token", json={"provider": "tikhub", "token": "th-key"}) + assert r.status_code == 200, r.text + + listed = [c for c in (await clients.get("/connections")).json() if c["provider"] == "tikhub"] + assert len(listed) == 1 and listed[0]["kind"] == "env" + assert (await clients.delete(f"/connections/{listed[0]['id']}")).status_code == 200 + + +# ---- HTTP Basic providers: paste a raw pair OR a ready-made Base64 blob ---------------------- +async def test_basic_provider_encodes_a_raw_login_password_once(clients: AsyncClient, monkeypatch): + """DataForSEO/Moz take HTTP Basic. Pasting the raw `login:password`, treg Base64s it once and the + upstream sees `Basic `.""" + import base64 + monkeypatch.setitem(P.REGISTRY, "dataforseo", dataclasses.replace( + P.REGISTRY["dataforseo"], base_url="http://upstream", probe_path="/whoami")) + r = await clients.post("/connections/token", json={"provider": "dataforseo", "token": "login:pw"}) + assert r.status_code == 200, r.text + echoed = (await clients.get("/call/dataforseo/whoami")).json()["auth"] + assert echoed == "Basic " + base64.b64encode(b"login:pw").decode() + + +async def test_basic_provider_accepts_a_ready_made_base64_blob(clients: AsyncClient, monkeypatch): + """The DataForSEO and Moz dashboards ALSO hand out a ready-made Base64 credential, and users paste + that at least as often as the raw pair. treg must NOT Base64 it a second time — the upstream has to + receive exactly `Basic `, decoding back to the original `login:password`.""" + import base64 + blob = base64.b64encode(b"login:pw").decode() + monkeypatch.setitem(P.REGISTRY, "dataforseo", dataclasses.replace( + P.REGISTRY["dataforseo"], base_url="http://upstream", probe_path="/whoami")) + r = await clients.post("/connections/token", json={"provider": "dataforseo", "token": blob}) + assert r.status_code == 200, r.text + echoed = (await clients.get("/call/dataforseo/whoami")).json()["auth"] + assert echoed == "Basic " + blob, "a pasted Base64 blob must not be double-encoded" + + +# ---- corrected probe shapes (regression guards for the 2026-08-13 connect-flow fixes) -------- +def test_brightdata_probe_is_a_real_route(): + """The old /datasets/v3/datasets 404'd even for a valid token, refusing every real key. /status is + the free account check that answers 200 (valid) / 401 (bad).""" + assert P.get("brightdata").probe_path == "/status" + + +def test_justoneapi_probe_uses_camelcase_uniqueid(): + """snake_case unique_id made the API answer HTTP 400 ('must input one of them (uniqueId or + secUid)') and a VALID token was refused. The param is camelCase.""" + assert "uniqueId=" in P.get("justoneapi").probe_path + assert "unique_id=" not in P.get("justoneapi").probe_path + + +async def test_probe_parses_json_body_labelled_text_plain(clients: AsyncClient, monkeypatch): + """ScrapeCreators answers HTTP 200 with a JSON body but a text/plain content-type; validity lives + in creditCount. Gating the parse on application/json left the field unread and refused a good key.""" + monkeypatch.setitem(P.REGISTRY, "scrapecreators", dataclasses.replace( + P.REGISTRY["scrapecreators"], base_url="http://upstream", + probe_path="/credit-json-as-text", token_verify_field="creditCount")) + r = await clients.post("/connections/token", json={"provider": "scrapecreators", "token": "sc-key"}) + assert r.status_code == 200, r.text + + +async def test_probe_keeps_a_query_string_baked_into_probe_path(clients: AsyncClient, monkeypatch): + """A probe_path like `/autocomplete?field=title` must reach the upstream WITH that query — httpx + drops a URL's own query when params= is passed, which used to 400 the probe and refuse the key.""" + monkeypatch.setitem(P.REGISTRY, "pdl", dataclasses.replace( + P.REGISTRY["pdl"], base_url="http://upstream", + probe_path="/needs-query?field=title&text=data", token_verify_field="")) + r = await clients.post("/connections/token", json={"provider": "pdl", "token": "pdl-key"}) + assert r.status_code == 200, r.text + + +async def test_query_token_survives_alongside_a_probe_path_query(clients: AsyncClient, monkeypatch): + """A query-credential provider (SpyFu: ?api_key=) whose probe_path ALSO carries a required query + (?domain=) must send both — the merge keeps the path's params and adds the credential on top.""" + monkeypatch.setitem(P.REGISTRY, "spyfu", dataclasses.replace( + P.REGISTRY["spyfu"], base_url="http://upstream", + probe_path="/needs-query?field=title", token_verify_field="")) + r = await clients.post("/connections/token", json={"provider": "spyfu", "token": "spyfu-secret"}) + assert r.status_code == 200, r.text diff --git a/tests/test_oauth_providers_m3.py b/tests/test_oauth_providers_m3.py index 4e7983307..cda678cd6 100644 --- a/tests/test_oauth_providers_m3.py +++ b/tests/test_oauth_providers_m3.py @@ -1 +1,367 @@ -PLACEHOLDER_TEST_OAUTH \ No newline at end of file +"""Milestone 3 — the rest of the Google family, Slack, and X. + +X is the interesting one: it rejects an authorization code exchanged without a PKCE verifier, and +rejects the client secret in the request body. Both quirks are captured on the pending connect at +start time so the callback exchanges the code exactly the way the consent URL was built. +""" + +from __future__ import annotations + +import json +from dataclasses import replace +from urllib.parse import parse_qs, urlsplit + +import pytest +from httpx import AsyncClient +from sqlmodel import select + +from treg import oauth +from treg import oauth_providers as P +from treg.config import get_settings +from treg.infra.db import session_maker +from treg.models import PendingOAuth + + +@pytest.fixture +def all_apps(monkeypatch): + for k in ("GOOGLE", "SLACK", "X", "TIKTOK"): + monkeypatch.setenv(f"TREG_{k}_CLIENT_ID", f"{k.lower()}-cid") + monkeypatch.setenv(f"TREG_{k}_CLIENT_SECRET", f"{k.lower()}-csec") + monkeypatch.setenv("TREG_META_CLIENT_ID", "meta-cid") + monkeypatch.setenv("TREG_META_CLIENT_SECRET", "meta-csec") + monkeypatch.setenv("TREG_INSTAGRAM_CLIENT_ID", "instagram-cid") + monkeypatch.setenv("TREG_INSTAGRAM_CLIENT_SECRET", "instagram-csec") + get_settings.cache_clear() + yield + get_settings.cache_clear() + + +def _q(payload: dict) -> dict: + return parse_qs(urlsplit(payload["consent_url"]).query) + + +# ---- registry shape ---------------------------------------------------------------------- +def test_every_provider_is_registered(): + assert set(P.REGISTRY) == { + "google-search-console", "google-analytics", "google-business-profile", "google-tag-manager", + "google-ads", "youtube", "linkedin", "slack", "x", "tiktok", + "facebook", "instagram", "meta-ads", + # API-key providers (auth_kind="key") + "apollo", "pdl", "akta", "hunter", "feedjolt", "crunchbase", "tikhub", "brightdata", "semrush", "justoneapi", + "scrapecreators", + "dataforseo", "seranking", "moz", "majestic", "serpstat", "exa", + "lusha", "coresignal", "diffbot", "thecompaniesapi", "leadmagic", "fiber-ai", + "companyenrich", "oceanio", "tomba", "predictleads", "findymail", "branddev", + "icypeas", "leadsforge", "influencersclub", "crustdata", "aviato", + "spyfu", "apify", "meta-ad-library", "serpapi", + "coingecko", "polygon", "finnhub", "twelvedata", "fmp", "eodhd", "marketstack", "tiingo", + "microsoft-ads", "snapchat-ads", "tiktok-ads", "pinterest-ads", + } + + +def test_default_capability_is_the_broadest(): + """Connect asks for the fullest capability; a narrower one is chosen up front, not bolted on + afterwards. Every provider's write must be a superset of its read for that to be safe.""" + assert P.GOOGLE_SEARCH_CONSOLE.default_capability == "write" + assert P.X.default_capability == "write" + assert P.GOOGLE_ADS.default_capability == "manage" # it has no read-only mode + assert P.GOOGLE_TAG_MANAGER.default_capability == "manage" + for provider in P.REGISTRY.values(): + caps = provider.capabilities + if "read" in caps and "write" in caps: + assert set(provider.scopes["read"]) < set(provider.scopes["write"]), provider.service + + +def test_google_ads_refuses_to_autoprovision(): + """Ads needs a developer-token header too; a bearer-only tool would 401 on first use.""" + assert P.GOOGLE_ADS.can_autoprovision is False + assert "developer-token" in P.GOOGLE_ADS.extra_credential_note + assert P.GOOGLE_SEARCH_CONSOLE.can_autoprovision is True + + +def test_x_write_keeps_offline_access(): + """Without offline.access the token can't be refreshed and every X connection becomes a + manual-reconnect chore within hours.""" + assert "offline.access" in P.X.scopes_for("write") + assert "offline.access" in P.X.scopes_for("read") + + +# ---- X's two quirks ---------------------------------------------------------------------- +async def test_x_consent_url_carries_a_pkce_challenge(clients: AsyncClient, all_apps): + d = (await clients.post("/oauth/start", json={"provider": "x"})).json() + q = _q(d) + assert q["code_challenge_method"] == ["S256"] + assert q["code_challenge"], "X rejects a code exchanged without a verifier" + # the verifier itself must stay server-side + assert "code_verifier" not in q + + +async def test_pkce_challenge_matches_the_stored_verifier(clients: AsyncClient, all_apps): + d = (await clients.post("/oauth/start", json={"provider": "x"})).json() + challenge = _q(d)["code_challenge"][0] + async with session_maker() as db: + p = (await db.execute(select(PendingOAuth).where(PendingOAuth.state == d["state"]))).scalars().one() + assert p.code_verifier + assert oauth.pkce_challenge(p.code_verifier) == challenge + assert p.token_endpoint_auth_method == "client_secret_basic" + + +async def test_google_does_not_use_pkce(clients: AsyncClient, all_apps): + d = (await clients.post("/oauth/start", json={"provider": "google-search-console"})).json() + assert "code_challenge" not in _q(d) + + +# ---- TikTok's two quirks ------------------------------------------------------------------- +async def test_tiktok_consent_url_uses_client_key_not_client_id(clients: AsyncClient, all_apps): + """TikTok ignores the OAuth2 spelling. Sending `client_id` gets a consent page that errors out + rather than an obvious 400, so this is worth pinning.""" + q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) + assert q["client_key"] == ["tiktok-cid"] + assert "client_id" not in q + + +async def test_tiktok_comma_joins_its_scopes(clients: AsyncClient, all_apps): + """Space-joined scopes come back from TikTok as scope_not_authorized — it splits on commas.""" + q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) + scope = q["scope"][0] + assert "," in scope and " " not in scope + assert set(scope.split(",")) == set(P.TIKTOK.scopes_for(P.TIKTOK.default_capability)) + + +async def test_tiktok_granted_scopes_are_stored_space_joined(clients: AsyncClient, all_apps, monkeypatch): + """The wire dialect must not leak into storage: every reader of granted_scopes uses .split(), + so a comma-joined grant would read as one bogus scope and report every capability unsatisfied.""" + # Registry mode takes token_uri from the provider, not the body, so point the provider itself at + # the in-process upstream (frozen dataclass → replace rather than setattr). + monkeypatch.setitem(P.REGISTRY, "tiktok", replace(P.TIKTOK, token_uri="http://upstream/token")) + body = {"provider": "tiktok", "capability": "post"} + state = (await clients.post("/oauth/start", json=body)).json()["state"] + await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") + sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] + + conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] + assert set(conn["capabilities"]) == {"read", "draft", "post"} + + +def test_tiktok_capabilities_are_cumulative(): + """draft must contain read and post must contain draft, or satisfied_capabilities() (which is + set-containment) reports a connection that can post but cannot read.""" + t = P.TIKTOK + assert set(t.scopes_for("read")) < set(t.scopes_for("draft")) < set(t.scopes_for("post")) + assert t.default_capability == "post" + # video.publish is the whole difference between "we drafted it for you" and "we posted it". + assert "video.publish" not in t.scopes_for("draft") + assert "video.publish" in t.scopes_for("post") + + +# ---- per-provider consent params --------------------------------------------------------- +async def test_google_keeps_offline_consent_params(clients: AsyncClient, all_apps): + """access_type=offline + prompt=consent is what guarantees Google returns a refresh_token.""" + q = _q((await clients.post("/oauth/start", json={"provider": "google-search-console"})).json()) + assert q["access_type"] == ["offline"] and q["prompt"] == ["consent"] + + +def test_slack_is_bring_your_own_bot(): + """A Slack bot is workspace-scoped and belongs to the workspace it's installed in. A shared + treg app would sit between a team and their own messages — and couldn't be installed on their + behalf anyway — so the user brings their own token instead of consenting to ours.""" + assert P.SLACK.auth_kind == "token" + assert P.SLACK.scopes == {}, "no consent screen means no capability sizing" + assert P.SLACK.default_capability == "", "and nothing to default to" + + +async def test_each_provider_uses_its_own_client_credentials(clients: AsyncClient, all_apps): + for service, expected in (("google-search-console", "google-cid"), + ("google-tag-manager", "google-cid"), ("x", "x-cid")): + q = _q((await clients.post("/oauth/start", json={"provider": service})).json()) + assert q["client_id"] == [expected], service + + +# ---- scope gap detection (the re-consent trigger) ----------------------------------------- +def test_satisfied_capabilities_detects_a_scope_gap(): + """Providers never backfill scopes onto an issued grant — a later capability needs re-consent, + and this is how we know to prompt instead of letting the call 403.""" + gsc = P.GOOGLE_SEARCH_CONSOLE + read_only = gsc.scopes_for("read") + assert gsc.satisfied_capabilities(read_only) == ["read"] + assert "write" not in gsc.satisfied_capabilities(read_only) + both = read_only + gsc.scopes_for("write") + assert set(gsc.satisfied_capabilities(both)) == {"read", "write"} + + +def test_google_tag_manager_capabilities_are_cumulative_and_exclude_admin(): + """GTM can audit, prepare and publish without authority to delete an entire container or + administer the account's users. Each wider tier must still satisfy every narrower tier.""" + gtm = P.GOOGLE_TAG_MANAGER + assert set(gtm.scopes_for("read")) < set(gtm.scopes_for("write")) < set(gtm.scopes_for("manage")) + assert gtm.default_capability == "manage" + requested = {scope for scopes in gtm.scopes.values() for scope in scopes} + assert not { + "https://www.googleapis.com/auth/tagmanager.delete.containers", + "https://www.googleapis.com/auth/tagmanager.manage.users", + "https://www.googleapis.com/auth/tagmanager.manage.accounts", + } & requested + assert gtm.probe_path == gtm.discover_path == "/tagmanager/v2/accounts" + assert gtm.discover_key == "account" + assert gtm.discover_id_field == "path" + assert gtm.discover_label_field == "name" + + +async def test_unconfigured_providers_are_listed_but_flagged(clients: AsyncClient, monkeypatch): + monkeypatch.setenv("TREG_X_CLIENT_ID", "") + monkeypatch.setenv("TREG_X_CLIENT_SECRET", "") + get_settings.cache_clear() + try: + rows = {p["service"]: p for p in (await clients.get("/oauth/providers")).json()} + assert rows["x"]["configured"] is False + r = await clients.post("/oauth/start", json={"provider": "x"}) + assert r.status_code == 422 and "not configured" in r.text + # a bring-your-own-token provider needs nothing from the deployment, so it stays offerable + assert rows["slack"]["configured"] is True + finally: + get_settings.cache_clear() + + +# ---- the gap, surfaced on the connection -------------------------------------------------- +async def test_connection_reports_the_capability_it_lacks(clients: AsyncClient, all_apps): + """Connect read-only, then see that `write` is named as missing — the reconnect trigger.""" + body = { + "provider": "google-search-console", "capability": "read", + "token_uri": "http://upstream/token", # the in-process upstream stands in for Google + } + state = (await clients.post("/oauth/start", json=body)).json()["state"] + await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") + sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] + + conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] + assert conn["capabilities"] == ["read"] + assert conn["missing_capabilities"] == ["write"] + + +async def test_byo_connection_has_no_capability_fields(clients: AsyncClient): + body = {"name": "byo", "client_id": "c", "client_secret": "s", + "auth_uri": "http://p/auth", "token_uri": "http://upstream/token", "scopes": ["x"]} + state = (await clients.post("/oauth/start", json=body)).json()["state"] + await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") + sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] + conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] + assert "missing_capabilities" not in conn # nothing to compare against without a provider + + +# ---- LinkedIn ----------------------------------------------------------------------------- +def test_linkedin_has_one_capability(): + """These scopes let a member read their own profile and post as themselves. A read-only + LinkedIn connection could do nothing but identify you, so there is no second option worth + asking about — and a dialog with one real choice is just friction.""" + assert P.LINKEDIN.capabilities == ["write"] + assert "w_member_social" in P.LINKEDIN.scopes_for("write") + + +async def test_linkedin_does_not_get_googles_consent_params(clients: AsyncClient, monkeypatch): + monkeypatch.setenv("TREG_LINKEDIN_CLIENT_ID", "li-cid") + monkeypatch.setenv("TREG_LINKEDIN_CLIENT_SECRET", "li-csec") + get_settings.cache_clear() + try: + d = (await clients.post("/oauth/start", json={"provider": "linkedin"})).json() + q = _q(d) + assert q["client_id"] == ["li-cid"] + assert "access_type" not in q and "prompt" not in q, "LinkedIn rejects Google's params" + assert "w_member_social" in q["scope"][0] + finally: + get_settings.cache_clear() + + +def test_instagram_direct_and_page_grants_use_explicit_app_profiles(): + assert P.INSTAGRAM.client_id_setting == "instagram_client_id" + parsed = urlsplit(P.INSTAGRAM.base_url) + assert parsed.scheme == "https" + assert parsed.hostname == "graph.instagram.com" + page = P.INSTAGRAM.profile_for_authorization("facebook-page") + assert page.client_id_setting == P.FACEBOOK.client_id_setting == "meta_client_id" + assert page.base_url == P.FACEBOOK.base_url + + +def test_meta_capabilities_are_cumulative(): + """satisfied_capabilities() is set containment, so a non-cumulative tier would report a + connection that can publish but 'cannot read' — and the default capability would be wrong. + default_capability is the BROADEST tier by design (one honest consent screen beats + connect-twice), so adding manage moved the default there.""" + for provider in (P.FACEBOOK, P.INSTAGRAM): + assert set(provider.scopes["read"]) < set(provider.scopes["post"]), provider.service + assert set(provider.scopes["post"]) < set(provider.scopes["manage"]), provider.service + assert provider.default_capability == "manage", provider.service + + +def test_meta_messaging_stays_out_of_the_publish_tier(): + """A publish-only connect must never put "manage your messages" (or lead retrieval, or the + Page's Messenger inbox) on the consent screen — the two-way surfaces live only in manage.""" + two_way = { + "instagram_manage_messages", "instagram_manage_comments", "pages_messaging", + "pages_manage_engagement", "leads_retrieval", "catalog_management", + } + for provider in (P.FACEBOOK, P.INSTAGRAM): + for cap in ("read", "post"): + assert not two_way & set(provider.scopes[cap]), (provider.service, cap) + assert "instagram_business_manage_messages" in P.INSTAGRAM.scopes["manage"] + assert {"instagram_manage_messages", "pages_messaging"} <= set(P.INSTAGRAM.scopes["page-tools"]) + + +def test_lead_retrieval_brings_its_required_rider(): + """Meta only honors leads_retrieval alongside pages_manage_ads — requesting one without the + other consents fine and then 400s on /leads, which would demo as a broken integration.""" + manage = set(P.FACEBOOK.scopes["manage"]) + assert {"leads_retrieval", "pages_manage_ads"} <= manage + + +def test_instagram_login_is_direct_and_page_discovery_is_optional(): + for cap in ("read", "post", "manage"): + assert "pages_show_list" not in P.INSTAGRAM.scopes[cap] + assert P.INSTAGRAM.identity_required is True + page = P.INSTAGRAM.profile_for_authorization("facebook-page") + assert page.discover_id_field == "instagram_business_account.id" + assert "pages_show_list" in P.INSTAGRAM.scopes["page-tools"] + + +def test_meta_asks_for_a_long_lived_token(): + """Meta's code exchange yields a ~1-2h token and no refresh_token. Without the second + exchange every Meta connection dies the day it is made.""" + assert P.FACEBOOK.long_lived_exchange + assert P.INSTAGRAM.long_lived_exchange_style == "instagram" + assert not P.TIKTOK.long_lived_exchange # nothing else should have picked it up + + +def test_instagram_consent_never_mentions_page_publishing(): + """Scopes are per capability. An Instagram connect asking for pages_manage_posts would put + 'manage your Pages' posts' on the consent screen for authority it never uses.""" + for cap in P.INSTAGRAM.scopes.values(): + assert "pages_manage_posts" not in cap + + +def test_meta_page_discovery_can_walk_the_business_graph(): + """Most agency-held Pages (and the Instagram accounts linked to them) are OWNED by a Business + portfolio, where the member has business-level access and no personal Page role. Drop + business_management from either capability and that user consents cleanly, then gets an empty + picker — the extra listing 400s and is (rightly) swallowed.""" + for cap in P.FACEBOOK.scopes.values(): + assert "business_management" in cap + page = P.INSTAGRAM.profile_for_authorization("facebook-page") + assert "business_management" in P.INSTAGRAM.scopes["page-tools"] + for provider in (P.FACEBOOK, page): + assert provider.discover_extra_path.startswith("/me/businesses"), provider.service + assert provider.discover_extra_list_paths, provider.service + + +def test_meta_ads_needs_no_second_credential(): + """Google Ads is gated on a developer token from an approved manager account; Meta has no + equivalent, so a Meta Ads connect must yield a callable tool on its own.""" + assert P.META_ADS.can_autoprovision is True + assert P.META_ADS.needs_extra_credential is False + + +def test_meta_ads_read_can_still_list_accounts(): + """/me/adaccounts is a Business asset listing. Drop business_management from read and the + connect consents cleanly, then offers an empty account picker.""" + for cap in P.META_ADS.scopes.values(): + assert "business_management" in cap + assert set(P.META_ADS.scopes["read"]) < set(P.META_ADS.scopes["manage"]) + assert "ads_management" not in P.META_ADS.scopes["read"], "read must not be able to spend money" From cd646ca97c2be73dac5276760ac61212edb70031 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 14:39:58 +0200 Subject: [PATCH 05/31] Register Feedjolt as a BYOK catalog provider. --- render.yaml | 360 +++++++++++++++++++++++++++++++++- src/treg/config.py | 480 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 838 insertions(+), 2 deletions(-) diff --git a/render.yaml b/render.yaml index 60e950c7b..36ffb0aef 100644 --- a/render.yaml +++ b/render.yaml @@ -1 +1,359 @@ -PLACEHOLDER_RENDER \ No newline at end of file +# Render Blueprint — deploys treg as one web service (API + dashboard + landing + tutorial + +# /llms.txt + /install.sh) backed by a managed Postgres. See docs/context/ops/deploy.md. +# +# The app is a single FastAPI service: `python -m treg` honors $PORT (Render routes + health-checks +# it). Web assets ship in the wheel via pyproject `force-include src/treg/web`, so `pip install .[server]` +# bundles everything — no separate frontend build. The `[server]` extra pulls the FastAPI/DB/crypto stack +# (the base install is the CLI only — see pyproject). +# +# Secrets (Fernet key, OAuth, Resend, admin/session tokens) are NOT in this file — they are set once +# in the Render dashboard (sync:false below marks them as dashboard-managed). The Postgres URL is +# auto-wired via fromDatabase; config.py rewrites postgres:// → postgresql+asyncpg:// at load. + +databases: + - name: treg-db + databaseName: treg + region: oregon + plan: basic-256mb # smallest paid tier with persistence; bump later if needed + +services: + - type: web + name: treg + runtime: python + region: oregon + plan: starter + branch: main + buildCommand: pip install ".[server]" + preDeployCommand: python -m treg upgrade + startCommand: python -m treg + healthCheckPath: /meta + autoDeploy: true + envVars: + # Postgres — auto-injected from the managed DB above (config.py adds the asyncpg driver). + - key: TREG_DATABASE_URL + fromDatabase: + name: treg-db + property: connectionString + # Public base URL — drives the OAuth callback, /meta, /llms.txt, /install.sh, all {BASE} links. + # CUTOVER (treg.superdesign.dev → treg.to, 2026-08): setting this back to + # https://treg.superdesign.dev is the complete, lossless rollback — PUBLIC_HOST_ALIASES + # keeps both names valid in both directions, and the old domain must stay attached on + # Render forever (installed clients hold tokens against it). + - key: TREG_PUBLIC_URL + value: https://treg.to + # Never return OTP codes in prod responses (account-takeover vector); email them via Resend. + - key: TREG_EMAIL_DEV_MODE + value: "false" + # treg.to is Resend-verified (DKIM + SPF). Deliverability canary after every change to this + # value: sign in via email OTP and confirm the code ARRIVES — send failures are swallowed by + # design, so a broken sender silently kills OTP login. Rollback: the old sender stays + # verified in Resend forever. + - key: TREG_EMAIL_FROM + value: tools-registry + - key: PYTHON_VERSION + value: "3.12.7" + # Dashboard-managed secrets (paste values in Render; sync:false keeps them out of git). + - key: TREG_SECRET_KEY # Fernet key — MUST equal local .env verbatim (irreplaceable). + sync: false + - key: TREG_SESSION_SECRET + sync: false + - key: TREG_ADMIN_TOKEN + sync: false + - key: TREG_GITHUB_CLIENT_ID + sync: false + - key: TREG_GITHUB_CLIENT_SECRET + sync: false + - key: TREG_RESEND_API_KEY + sync: false + # Support chat (Intercom Messenger, treg's own workspace). Unset = widget off everywhere. + - key: TREG_INTERCOM_APP_ID # public workspace id (visible in page source anyway) + sync: false + - key: TREG_INTERCOM_SECRET # identity-verification secret — signs user_hash, never sent to the browser + sync: false + # Landing live-wire demo (optional — unset = sandbox calls all synthesize / webhook 404s). + - key: TREG_DEMO_STRIPE_KEY # Stripe sandbox restricted key (Charges only) for the live wire + sync: false + - key: TREG_DEMO_STRIPE_WEBHOOK_SECRET # whsec_… from the DEMO Stripe sandbox's webhook endpoint + sync: false + # Tier-4 platform keys: treg's OWN provider credentials, spent on a caller's behalf and metered + # against their prepaid balance (docs/PLATFORM-BALANCE-PLAN.md §Phase 3). A key alone does + # nothing — TREG_PLATFORM_PROVIDERS is the allow-list AND the kill switch: set it to "" to turn + # tier 4 off instantly, without a redeploy. Fund each provider account upstream first. + - key: TREG_PLATFORM_KEY_TIKHUB # TikHub API key (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_DATAFORSEO # base64 of "login:password" (HTTP Basic) + sync: false + - key: TREG_PLATFORM_KEY_SCRAPECREATORS # ScrapeCreators API key (x-api-key) + sync: false + - key: TREG_PLATFORM_KEY_BRIGHTDATA # Bright Data account API token (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_JUSTONEAPI + sync: false + - key: TREG_PLATFORM_KEY_SERPAPI + sync: false + - key: TREG_PLATFORM_KEY_MOZ # base64 of "access_id:secret_key" (HTTP Basic) + sync: false + - key: TREG_PLATFORM_KEY_SERANKING + sync: false + - key: TREG_PLATFORM_KEY_HUNTER + sync: false + - key: TREG_PLATFORM_KEY_FEEDJOLT # plan-included; BYOK until a per-call rate card exists + sync: false + - key: TREG_PLATFORM_KEY_LEADMAGIC + sync: false + - key: TREG_PLATFORM_KEY_LUSHA + sync: false + - key: TREG_PLATFORM_KEY_PDL + sync: false + - key: TREG_PLATFORM_KEY_DIFFBOT + sync: false + - key: TREG_PLATFORM_KEY_AKTA + sync: false + - key: TREG_PLATFORM_KEY_APIFY + sync: false + - key: TREG_PLATFORM_KEY_SERPSTAT # Serpstat API token (?token=…) + sync: false + - key: TREG_PLATFORM_KEY_SPYFU # SpyFu SECRET KEY alone (?api_key=…), not id/base64 + sync: false + - key: TREG_PLATFORM_KEY_CORESIGNAL # Coresignal API key (`apikey` header) + sync: false + - key: TREG_PLATFORM_KEY_THECOMPANIESAPI # raw token, injected as "Basic {secret}" un-encoded + sync: false + - key: TREG_PLATFORM_KEY_COMPANYENRICH # CompanyEnrich API key (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_OCEANIO # Ocean.io API token (X-Api-Token); tier 4 refused until fx.yaml prices it + sync: false + - key: TREG_PLATFORM_KEY_PREDICTLEADS # base64 of "api_key:api_token" (HTTP Basic) + sync: false + - key: TREG_PLATFORM_KEY_FINDYMAIL # Findymail API key (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_BRANDDEV # Brand.dev API key (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_ICYPEAS # Icypeas API key (raw Authorization header) + sync: false + - key: TREG_PLATFORM_KEY_LEADSFORGE # LeadsForge API key (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_INFLUENCERSCLUB # influencers.club dashboard API key, a JWT (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_CRUSTDATA # Crustdata API key; x-api-version is injected from provider metadata + sync: false + - key: TREG_PLATFORM_KEY_AVIATO # Aviato API key (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_EXA # Exa API key (x-api-key); every response reports costDollars + sync: false + - key: TREG_PLATFORM_KEY_FIBER_AI # Fiber AI key (x-api-key) + sync: false + - key: TREG_PLATFORM_KEY_TOMBA # Tomba API key ta_… (X-Tomba-Key) + sync: false + - key: TREG_PLATFORM_KEY_TOMBA_SECRET # Tomba API secret ts_… (X-Tomba-Secret) — set BOTH + sync: false + - key: TREG_PLATFORM_PROVIDERS # e.g. "tikhub,dataforseo,scrapecreators"; "" = tier 4 off + value: "" + # Overflow (docs/context/ops/capacity.md): treg-owned aggregator accounts serving the SAME vendor + # endpoint when our own account is out. MODE is the switch: off (default) | shadow | on. + - key: TREG_OVERFLOW_MODE + value: "off" + - key: TREG_OVERFLOW_DAILY_BUDGET_USD + value: "20" + - key: TREG_OVERFLOW_KEY_ORTHOGONAL + sync: false + - key: TREG_OVERFLOW_KEY_MONID + sync: false + # Worker profile: the hourly capacity sweep (docs/context/ops/capacity.md). Same code, same env + # as the web service (fromService — add a platform key in the web service only), no HTTP. + - type: cron + name: treg-capacity-sweep + runtime: python + region: oregon + plan: starter + branch: main + schedule: "17 * * * *" + buildCommand: pip install ".[server]" + startCommand: treg-worker capacity sweep + envVars: + - key: TREG_DATABASE_URL + fromDatabase: + name: treg-db + property: connectionString + - key: PYTHON_VERSION + value: "3.12.7" + - key: TREG_SECRET_KEY + fromService: + type: web + name: treg + envVarName: TREG_SECRET_KEY + - key: TREG_OVERFLOW_KEY_ORTHOGONAL + fromService: + type: web + name: treg + envVarName: TREG_OVERFLOW_KEY_ORTHOGONAL + - key: TREG_OVERFLOW_KEY_MONID + fromService: + type: web + name: treg + envVarName: TREG_OVERFLOW_KEY_MONID + - key: TREG_PLATFORM_KEY_TIKHUB + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_TIKHUB + - key: TREG_PLATFORM_KEY_DATAFORSEO + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_DATAFORSEO + - key: TREG_PLATFORM_KEY_SCRAPECREATORS + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_SCRAPECREATORS + - key: TREG_PLATFORM_KEY_BRIGHTDATA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_BRIGHTDATA + - key: TREG_PLATFORM_KEY_JUSTONEAPI + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_JUSTONEAPI + - key: TREG_PLATFORM_KEY_SERPAPI + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_SERPAPI + - key: TREG_PLATFORM_KEY_MOZ + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_MOZ + - key: TREG_PLATFORM_KEY_SERANKING + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_SERANKING + - key: TREG_PLATFORM_KEY_HUNTER + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_HUNTER + - key: TREG_PLATFORM_KEY_LEADMAGIC + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_LEADMAGIC + - key: TREG_PLATFORM_KEY_LUSHA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_LUSHA + - key: TREG_PLATFORM_KEY_PDL + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_PDL + - key: TREG_PLATFORM_KEY_DIFFBOT + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_DIFFBOT + - key: TREG_PLATFORM_KEY_AKTA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_AKTA + - key: TREG_PLATFORM_KEY_APIFY + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_APIFY + - key: TREG_PLATFORM_KEY_SERPSTAT + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_SERPSTAT + - key: TREG_PLATFORM_KEY_SPYFU + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_SPYFU + - key: TREG_PLATFORM_KEY_CORESIGNAL + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_CORESIGNAL + - key: TREG_PLATFORM_KEY_THECOMPANIESAPI + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_THECOMPANIESAPI + - key: TREG_PLATFORM_KEY_COMPANYENRICH + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_COMPANYENRICH + - key: TREG_PLATFORM_KEY_OCEANIO + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_OCEANIO + - key: TREG_PLATFORM_KEY_PREDICTLEADS + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_PREDICTLEADS + - key: TREG_PLATFORM_KEY_FINDYMAIL + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_FINDYMAIL + - key: TREG_PLATFORM_KEY_BRANDDEV + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_BRANDDEV + - key: TREG_PLATFORM_KEY_ICYPEAS + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_ICYPEAS + - key: TREG_PLATFORM_KEY_LEADSFORGE + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_LEADSFORGE + - key: TREG_PLATFORM_KEY_INFLUENCERSCLUB + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_INFLUENCERSCLUB + - key: TREG_PLATFORM_KEY_CRUSTDATA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_CRUSTDATA + - key: TREG_PLATFORM_KEY_AVIATO + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_AVIATO + - key: TREG_PLATFORM_KEY_EXA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_EXA + - key: TREG_PLATFORM_KEY_FIBER_AI + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_FIBER_AI + - key: TREG_PLATFORM_KEY_TOMBA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_TOMBA + - key: TREG_PLATFORM_KEY_TOMBA_SECRET + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_TOMBA_SECRET diff --git a/src/treg/config.py b/src/treg/config.py index d86328da9..2773fa41c 100644 --- a/src/treg/config.py +++ b/src/treg/config.py @@ -1 +1,479 @@ -PLACEHOLDER_CONFIG \ No newline at end of file +"""Settings — read once from env/.env. Keep it tiny and explicit.""" + +from __future__ import annotations + +from functools import lru_cache + +from pydantic import field_validator +from urllib.parse import urlsplit + +from pydantic_settings import BaseSettings, SettingsConfigDict + +# Every hostname the reference deployment has EVER answered to. treg moved +# treg.superdesign.dev → treg.to (2026-08); installed CLIs, skill.md files, .mcp.json configs and +# MCP OAuth grants exist against BOTH names, so both stay valid everywhere a host or audience is +# recognized — mcp.py's transport allow-lists, mcp_oauth's token audiences, api.py's login-callback +# anchoring — REGARDLESS of which one `public_url` currently points at. That symmetry is what makes +# a TREG_PUBLIC_URL revert a complete rollback: grants and logins minted on either name survive the +# flip in either direction. Self-hosters are unaffected: these only ADD accepted names, and none of +# them resolve to a self-hosted deployment. +PUBLIC_HOST_ALIASES: tuple[str, ...] = ("treg.superdesign.dev", "treg.to") + +# The subset browsers are REDIRECTED AWAY FROM (marketing pages only; see api.py's middleware). +# Deliberately one-way — only ever the pre-move name, never treg.to — so a browser that cached the +# old→new 301 can never meet a new→old redirect and loop, even while a rollback is in effect. +LEGACY_PUBLIC_HOSTS: tuple[str, ...] = ("treg.superdesign.dev",) + + +def platform_setting_name(provider: str) -> str: + """The Settings attribute holding treg's own key for `provider` — the string a `platform_setting` + binding carries, and the only form of a platform credential that ever leaves this module.""" + return "platform_key_" + (provider or "").lower().replace("-", "_") + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", env_prefix="TREG_", extra="ignore") + + # SQLite locally, Postgres on Render — same code path, just swap the URL. + database_url: str = "sqlite+aiosqlite:///./treg.db" + + @field_validator("database_url") + @classmethod + def _async_pg_driver(cls, v: str) -> str: + # Render's `fromDatabase` (render.yaml) injects a bare `postgres://`/`postgresql://` URL, but + # our async engine (create_async_engine) needs the asyncpg driver. Rewrite the scheme so the + # Blueprint can auto-wire the DB with no manual URL editing. No-op for sqlite / already-drivered URLs. + if v.startswith("postgres://"): + v = "postgresql://" + v[len("postgres://") :] + if v.startswith("postgresql://"): + v = "postgresql+asyncpg://" + v[len("postgresql://") :] + return v + + # Fernet key (urlsafe base64, 32 bytes). Generate with `treg keygen` (see crypto.py). + # Empty in dev means an ephemeral key is minted at startup (secrets won't survive a restart). + secret_key: str = "" + + # The single bootstrap caller token for the MVP. Per-user/org tokens come in Step 3. + api_token: str = "dev-token" + + # DEMO Stripe webhook signing secret (env TREG_DEMO_STRIPE_WEBHOOK_SECRET) for the landing page's live + # payments feed (see pubfeed.py). Empty = the /stripe/webhook endpoint is off (404). + demo_stripe_webhook_secret: str = "" + + # The Stripe sandbox restricted key (env TREG_DEMO_STRIPE_KEY) behind the landing sandbox's ONE + # live wire: a sandbox call to the exact seeded stripe tool relays for real with THIS key + # injected — the key never exists in any sandbox org (see sandbox.is_live_tool / api.call_tool). + # Empty = every sandbox call synthesizes, exactly as before the live wire existed. + demo_stripe_key: str = "" + + # Cross-tenant super-admin bearer (env TREG_ADMIN_TOKEN). Presenting it authorizes every + # /admin/* endpoint regardless of org. Empty = the env key is disabled (only is_superadmin + # users can reach /admin). Keep it long + secret; it sees ALL orgs. + admin_token: str = "" + + # Isolated-runner proof for `treg run --local` (env TREG_RUN_PROOF). A grant that would return a + # secret the CALLER does NOT own (a shared-key tool a member may run but not read) requires this + # value in the `X-Treg-Run-Proof` header — held ONLY by the root-installed treg-run runner, never + # by the member. Empty = shared-key local runs are refused (owned-secret runs still work). Set it + # on the server AND install it via `treg setup-local-run --run-proof` to enable shared local runs. + run_proof: str = "" + + # `treg run --server` allow-list. The server only executes an entrypoint that is a catalog-known CLI + # (stripe/gh/vercel/…) OR listed here (comma-separated) — so a member can't name `bash`/`python` and + # run arbitrary code as the server user. Extend it as new CLIs are approved. (Full filesystem/network + # isolation — the stronger fix — needs a container deploy and is a planned follow-up.) + run_allowed_bins: str = "" + + # Server-run resource limits (the DoS half of the server-run sandbox). Every `--server` run's child + # process gets these POSIX rlimits so a runaway or hostile CLI can't exhaust the host. On by default; + # a no-op where `resource` is unavailable (non-Unix). We deliberately do NOT cap address space or + # process count — a virtual-memory cap crashes Go-based CLIs (gh/stripe/doctl), and the per-user + # process cap is shared with the server itself. CPU-seconds + max-file-size + no-core-dumps are the + # safe, high-value guards. Set TREG_RUN_RLIMITS=false to disable entirely. + run_rlimits: bool = True + run_cpu_seconds: int = 300 # CPU time a single server run may burn (backstop to the wall timeout) + run_fsize_mb: int = 100 # largest single file a server run may write (disk-fill guard) + + # ---- prepaid balance (domain/money) ----------------------------------------------------------- + # Platform markup on a call served by a PLATFORM key, applied to the estimate at reserve time and + # to the observed cost at settle time. 0.0 = we charge exactly what the provider charges us. + # It lives in config (rather than being hardcoded) so turning margin on is a deploy setting, not + # a code change — and so the ledger records the rate that was in force for each call. + platform_margin: float = 0.0 + # The signup gift, in micro-USD (1e-6 USD): $1 buys ~1,600 catalog calls, enough for an agent to + # get real work done before it ever sees a payment form. Granted once, at org creation only. + promo_grant_micro: int = 1_000_000 + # Upstream HTTP timeout for a relayed call (the shared httpx client). Also the base of the hold + # reaper's cutoff: a hold older than call_timeout_s + hold_grace_s belongs to a call that can no + # longer be settling, so the reaper returns it (see ledger.reap_stale_holds). + # 180, not 30: real catalog upstreams routinely run long — BrightData sync scrapes ~20-35s + # (a 30s ceiling 502'd one live), and merchant routes have been observed at 9s→105s. + call_timeout_s: int = 180 + hold_grace_s: int = 60 + + # ---- referral program (referrals.py) -------------------------------------------------------- + # Flat bounties, not a percentage of top-ups. At 0% platform margin a percentage would be a + # permanent share of pass-through GMV, and — unlike a flat figure — it rewards farming in + # proportion to effort. Nobody builds a fake-account farm for $10; plenty would for 15% of an + # uncapped balance. All figures are micro-USD (1e-6 USD), like every other amount in the system. + # Symmetric on purpose: both sides get the same, so the offer is one sentence to explain and + # neither party can feel like the other got the better end of it. + referral_referrer_micro: int = 5_000_000 # $5 to the person who shared the link + referral_referred_micro: int = 5_000_000 # $5 to the friend who signed up + # The friend's first top-up must clear this for anything to be owed. Deliberately well above the + # bounties themselves: below it, buying the bonus with your own card is profitable. + referral_min_topup_micro: int = 10_000_000 # $10 + # Days between qualifying and the credit landing. This is the ONLY clawback window that exists: + # referral credit is promotional, burns first (ledger._KIND_ORDER) and is typically spent within + # days, so once granted it cannot be recovered. Short on purpose — a referral program that pays + # in 60 days does not feel like a reward, and the exposure at these amounts is tiny. + referral_hold_days: int = 7 + # Lifetime paid referrals per person. Anyone who wants more is an influencer, and that is a + # conversation with a contract and a bank transfer — not a self-serve link. The refusal IS the + # commercial conversation, the same posture as raising a daily cap. + referral_cap: int = 20 + + # ---- tier-4 platform keys (api.py credential ladder) --------------------------------------- + # treg's OWN provider keys, spent on a caller's behalf and metered against their prepaid balance. + # Read by `proxy.relay` through a `platform_setting` binding (never copied into an org's secrets, + # never reachable from a local run). Naming is load-bearing: the binding names the ATTRIBUTE, so + # `platform_key_for("tikhub")` → `platform_key_tikhub` → env `TREG_PLATFORM_KEY_TIKHUB`. + # dataforseo's value is the base64 of "login:password" (HTTP Basic) — the same bytes a pasted + # secret ends up as; the other two are the raw key. + platform_key_tikhub: str = "" + platform_key_dataforseo: str = "" + platform_key_scrapecreators: str = "" + platform_key_brightdata: str = "" + platform_key_justoneapi: str = "" + platform_key_serpapi: str = "" + platform_key_moz: str = "" # base64 of "access_id:secret_key" (HTTP Basic) + platform_key_seranking: str = "" + platform_key_hunter: str = "" + platform_key_feedjolt: str = "" # Bearer fjk_…; plan-included REST, no per-call meter + platform_key_leadmagic: str = "" + platform_key_lusha: str = "" + platform_key_pdl: str = "" + platform_key_diffbot: str = "" + platform_key_akta: str = "" + platform_key_apify: str = "" + platform_key_serpstat: str = "" + platform_key_spyfu: str = "" # the SpyFu *secret key* alone (?api_key=…), not the id or base64 pair + platform_key_coresignal: str = "" + platform_key_thecompaniesapi: str = "" # raw token — injected as "Basic {secret}" un-encoded + platform_key_apollo: str = "" # raw key (X-Api-Key); billed at the Basic-plan $/credit rate in fx.yaml + # ---- Market data (2026-08-15). Two PAID plans billed per call, three FREE-tier keys serving + # capped trial pools at $0 (fx.yaml kind: treg_trial — api._enforce_trial_allowance is the brake). + platform_key_coingecko: str = "" # PRO key (x-cg-pro-api-key); Basic $29/mo, $0.00029/credit + platform_key_marketstack: str = "" # access_key; Basic $9.99/mo, $0.000999/call vs the 10k/mo cap + platform_key_finnhub: str = "" # FREE-tier key — trial pool, 50 calls/team/day + platform_key_twelvedata: str = "" # FREE Basic key (800/day TOTAL) — trial pool, 20 calls/team/day + platform_key_tiingo: str = "" # FREE Starter key (1,000/day total) — trial pool, 20 calls/team/day + # ---- Enrichment expansion (2026-08-20). Slots only — fund the accounts and set the keys before + # naming any of these in TREG_PLATFORM_PROVIDERS. + platform_key_companyenrich: str = "" # Bearer key + platform_key_oceanio: str = "" # X-Api-Token; fx.yaml usd is null so tier 4 stays refused until priced + platform_key_predictleads: str = "" # base64 of "api_key:api_token" (HTTP Basic, like dataforseo) + platform_key_findymail: str = "" # Bearer key + platform_key_branddev: str = "" # Bearer key + platform_key_icypeas: str = "" # raw key (Authorization, no Bearer) + platform_key_leadsforge: str = "" # Bearer key + platform_key_fiber_ai: str = "" # Fiber AI key (x-api-key header, sk_live_…) + platform_key_tomba: str = "" # the API key (ta_…); X-Tomba-Key header + platform_key_tomba_secret: str = "" # the API secret (ts_…); X-Tomba-Secret — BOTH must be set + # (tomba's data routes need the header pair; TOMBA.platform_extra_setting names this second slot) + platform_key_influencersclub: str = "" # Bearer key (dashboard JWT); creator discovery + enrichment, fx.yaml $0.598/credit (our $299/500 plan) + platform_key_crustdata: str = "" # Bearer key; every call also needs the pinned x-api-version header + platform_key_aviato: str = "" # Bearer key; $10 auto-top-up buys 1,000 credits + platform_key_exa: str = "" # x-api-key; dollar-metered ($7/1k searches, $1/1k pages); settles from costDollars.total + # Overflow aggregators (docs/PROVIDER-CAPACITY-PLAN.md §4.3): treg-owned accounts that serve the + # SAME vendor endpoint when our direct account is out. Env only, never a Secret row, never logged. + # Not platform_key_* on purpose: they are a credential RUNG (platform-overflow), not a provider. + overflow_key_orthogonal: str = "" + overflow_key_monid: str = "" + # off (default) | shadow | on. Shadow: on a tier-4 capacity failure, call the aggregator anyway, + # log status/shape/cost, still return the vendor's own error and charge the caller nothing — + # treg pays the probe, bounded by the daily budget. On: the child cycle serves the caller. + overflow_mode: str = "off" + overflow_daily_budget_usd: float = 20.0 # per aggregator, per UTC day; crossing it skips overflow + # The KILL SWITCH, and the reason a key alone isn't enough: a provider serves tier 4 only if it is + # named here AND its key is set. Empty (the default) = tier 4 is entirely off, so a deploy that + # happens to hold a key can't start spending it by accident. `TREG_PLATFORM_PROVIDERS=""` in the + # Render dashboard turns the whole feature off without a redeploy. + platform_providers: str = "" + # Whether DISCOVERY steers to routed rows: `treg.` first in search, and a routed + # parent pulled in whenever one of its children matched. `off` leaves every routed endpoint + # callable and priced — only the steering stops, and search looks as it did before routing + # shipped. Same runtime-switch shape as `platform_providers`: flip it in the dashboard, no + # redeploy. It exists because "does the router answer well" and "should every agent be led to + # it by default" are separate questions, and the second one is answered by traffic, not by + # argument. + routed_discovery: str = "on" + # Per-org, per-UTC-day ceiling on tier-4 spend, and the CEILING a team may raise its own + # `Org.daily_cap_micro` to. Enforced FAIL-CLOSED (unlike the soft per-user call cap): a query + # error refuses the call rather than letting an unbounded amount of our money out. It is a + # blast-radius limit on a runaway agent or a mispriced catalog entry, not a billing control — + # the balance is what a team actually spends against. + # + # Raised 100 -> 500 on 2026-08-29. At 100 an ordinary day's work tripped it: a benchmark agent + # exploring the catalog spends ~$0.10 a query, and 26 of 32 briefs came back empty because every + # call after the ceiling 429'd — the team had $92 of balance and could not use it. The rail is + # still here, and it is still ours to raise per team; it just should not fire before a real + # workload does. + platform_daily_cap_usd: float = 500.0 + # OAuth providers whose UPSTREAM bill lands on treg's developer app rather than the connected + # user (X moved to pay-per-use in Feb 2026: the app owner is billed per resource read / per post + # written, whoever's token made the call). Calls through a registry connect of a provider named + # here are metered against the org's balance — the same reserve→settle path as tier 4. Same + # kill-switch shape as `platform_providers`: empty (the default) = those calls stay free, so a + # deploy must OPT IN to charging (`TREG_OAUTH_BILLED_PROVIDERS=x`). BYO-app connections + # (/oauth/start with the caller's own client_id) are never metered — their upstream bill is + # already theirs. + oauth_billed_providers: str = "" + + # ---- Stripe top-ups (billing.py) ----------------------------------------------------------- + # OUR billing account's keys. Deliberately NOT the `demo_stripe_*` pair above: that one belongs to + # the landing page's sandbox feed and its webhook secret signs a different endpoint. Empty secret + # key = top-ups are off (the /billing/* endpoints 503); empty webhook secret = the webhook 404s, + # so an unconfigured deploy exposes no unauthenticated POST surface (same posture as the demo). + stripe_secret_key: str = "" + stripe_webhook_secret: str = "" + # Top-up amounts, in whole USD (the ONLY place dollars appear — billing.py converts to micro-USD + # for the ledger and to integer cents for Stripe at the boundaries, and nothing in between sees a + # float). The $10 minimum is fee math, not policy: at 2.9% + $0.30 a $5 top-up loses 9% to fees, + # and the referral offer's qualifying amount is $10 already. + topup_min_usd: int = 10 + topup_default_usd: int = 10 + # Four presets plus "Other" (any whole amount ≥ min). Deliberately few and skewed big: with eight + # cards from $5 up nobody ever picked $100+, and repeat payers stayed flat ($10 → $10). + topup_presets: list[int] = [10, 50, 100, 200] + # Bonus credit on a MANUAL top-up, as {min_usd: percent}: the highest key ≤ the amount applies. + # It is promotional credit (a separate block that burns first and is never refundable), never + # purchased balance — see billing.bonus_for_topup. Automatic refills get no bonus. + topup_bonus_tiers: dict[int, int] = {10: 0, 50: 5, 100: 10, 200: 15} + # After each manual top-up the dashboard's preselected amount steps one preset up, but never past + # this — the ladder nudges $10 payers toward $50 without preselecting $200 at anyone. + topup_default_cap_usd: int = 50 + # Auto-top-up defaults, applied when an org enables it without naming its own numbers. The + # threshold is deliberately above the $1 promo grant's tail: at agent call rates a $2 floor is one + # burst away from empty, and an off-session charge takes seconds to land. + autotopup_default_threshold_usd: int = 5 + autotopup_default_amount_usd: int = 20 + # Hard guardrails on the off-session charge — the difference between "convenient" and "a runaway + # agent bills a card all night". Cap is per calendar month, cooldown is between attempts, and + # max_attempts counts CONSECUTIVE failures before auto-top-up disables itself. + autotopup_monthly_cap_usd: int = 100 + autotopup_cooldown_s: int = 3600 + autotopup_max_attempts: int = 3 + + # Call-time SSRF guard on the proxy: resolve the upstream host and refuse an internal target. On by + # default; the test suite disables it (its upstream is an in-process ASGI transport, not real DNS). + proxy_ssrf_check: bool = True + + # The archive's rollout switch (see treg/archive.py): "off" (default) | "shadow" (record + + # learn from metered platform responses, serve nothing) | "serve" (shadow + answer eligible + # fresh hits from the store). Any other value degrades to "off" — a typo must disable, never + # enable. Staged deliberately so production can sit in "shadow" while phase 0 measures. + archive_mode: str = "off" + # Bodies above this size are hash-counted but never stored (skipped whole, not truncated): + # the archive is for API JSON answers, not downloads. Statistics still record size_bytes. + archive_max_body_bytes: int = 2_000_000 + # What happens to an endpoint WITHOUT a judged `cache:` field (the founder's 2026-08-29 + # decision): "transient" keeps every answer body as short-lived cache; "forbidden" is the old + # keep-nothing posture. A judged forbidden (a licence that was read and says no) is always + # respected, and actions are never stored, whatever this says. + archive_default_policy: str = "transient" + # The refresh worker (serve mode only): how often it scans for due keys, and how many + # refresh calls ONE provider may spend per UTC day. A refresh is treg's own vendor spend with + # no caller attached, so the cap is the brake — 0 disables refreshing without touching serving. + archive_refresh_interval_s: int = 300 + archive_refresh_daily_cap: int = 50 + + # Additive Claude directory MCP. Default OFF so deploying code cannot publish a new connector + # surface before its production Inspector and custom-connector gates have passed. + claude_connector_enabled: bool = False + + # Browser-only OAuth/MCP test harness. It handles real grants and therefore belongs on local + # and staging deployments, not on the public product surface. Explicitly enable it where an + # engineer is testing the protocol end to end. + connect_demo_enabled: bool = False + + # treg's own public base URL — used to build the OAuth callback (must be whitelisted in the + # provider's OAuth app). Self-hosting? Set TREG_PUBLIC_URL to your deployment's URL. + public_url: str = "https://treg.to" + # Proof to OpenAI's plugin directory that we control this domain. The portal generates a token + # and fetches it from /.well-known/openai-apps-challenge; that endpoint must return THAT token + # and nothing else — not JSON, not a list. Empty (the default) leaves the route 404, which is the + # right answer for every deployment that is not ours. + openai_apps_challenge: str = "" + + # Human login via GitHub OAuth (dashboard sessions). Create a GitHub OAuth App with callback + # /auth/github/callback and set these; empty disables the GitHub button. + github_client_id: str = "" + github_client_secret: str = "" + # Signs the session cookie (HMAC). Falls back to secret_key if unset. Set a real value in prod. + session_secret: str = "" + # Overridable for tests; real GitHub by default. + github_authorize_url: str = "https://github.com/login/oauth/authorize" + github_token_url: str = "https://github.com/login/oauth/access_token" + github_api_url: str = "https://api.github.com" + + # Human login via Google OAuth (dashboard sessions). Create a Google "web" OAuth client with an + # authorized redirect of /auth/google/callback; empty disables the Google button. + # The SAME client also backs registry connects (oauth_providers.py) — Search Console, Analytics, + # Ads, Business Profile — which consent through /oauth/callback. Register both + # redirect URIs on it. Login asks for openid/email/profile; a connect asks only for the scopes + # its capability needs, so the two never share a consent screen. + google_client_id: str = "" + google_client_secret: str = "" + # Product analytics (dashboard onboarding + Try-it). Empty key = OFF, so self-hosted treg instances + # send nothing — only a deployment that sets TREG_POSTHOG_KEY reports. The key is a PUBLIC posthog + # ingestion key (safe to expose to the browser); host defaults to EU cloud. + posthog_key: str = "" + posthog_host: str = "https://eu.i.posthog.com" + # Intercom Messenger (support chat; treg's own workspace). Empty app_id = OFF, so self-hosted + # instances never load the widget. The app_id is public (visible in page source); the secret + # signs user_hash for identity verification and must never reach the browser. + intercom_app_id: str = "" + intercom_secret: str = "" + # Registry OAuth apps for the non-Google providers (oauth_providers.py). Empty = that provider + # is listed as unconfigured rather than failing part-way through a consent. + # treg's own Google Ads developer token, from OUR approved manager account. Ads needs it on + # every call ALONGSIDE the user's OAuth — it identifies the calling application, not the user, + # and grants no access to our ad accounts. Holding it centrally is the whole point: a user + # would otherwise wait weeks for Google to approve one of their own. + google_ads_developer_token: str = "" + # Google Ads consents through a DEDICATED OAuth client in its own Cloud project — NOT the shared + # google_client_id. A developer token is permanently welded to the first Cloud project it calls + # from, and the shared project is already paired to a different token, so Ads must use a client + # from the project its live token is paired with. Empty = Ads shows as unconfigured (correct — it + # can't work without this) rather than silently reusing the wrong client. + google_ads_client_id: str = "" + google_ads_client_secret: str = "" + # Google Ads conversion upload. This customer id and the refresh token below are both required; + # if either is empty the whole feature is OFF (tests stay inert and self-hosters send nothing) — + # the same gate shape analytics.py uses for posthog_key. + google_ads_customer_id: str = "" + # Manager (MCC) account used to access google_ads_customer_id. Optional for direct client auth; + # when present this is sent as login-customer-id. It cannot be inferred from Secret.resource_ref, + # which is the TARGET client account selected in discovery, not its manager. + google_ads_login_customer_id: str = "" + # treg's OWN long-lived refresh token for the Data Manager conversion uploader — a PLATFORM + # credential, obtained once, out of band, by an operator via the OAuth playground with scope + # https://www.googleapis.com/auth/datamanager. It is exchanged against `google_ads_client_id`/ + # `_secret` above (the refresh token is issued against that client, so it must be redeemed with + # it), never against a customer's own OAuth connection: the uploader ships treg's OWN marketing + # conversions to treg's OWN ad account, a different purpose from a customer connecting Ads to + # read their campaign data, and the two must not share a credential or a consent screen. Empty + # = the whole feature is OFF. See docs/context/architecture/ads-conversions.md. + ads_conv_refresh_token: str = "" + + linkedin_client_id: str = "" + linkedin_client_secret: str = "" + + slack_client_id: str = "" + slack_client_secret: str = "" + x_client_id: str = "" + x_client_secret: str = "" + # TikTok issues a separate app (and separate key/secret) per sandbox, so a dev deployment + # points at a sandbox client while production points at the reviewed one. + tiktok_client_id: str = "" + tiktok_client_secret: str = "" + # Facebook Login credentials. These continue to back Facebook Pages, Meta Ads, and the optional + # Page-based Instagram authorization method. + meta_client_id: str = "" + meta_client_secret: str = "" + # Business Login for Instagram has its own Instagram App ID and App Secret. Meta documents + # these under Instagram → API setup with Instagram login. They are not interchangeable with + # the Facebook Login app credentials above. + instagram_client_id: str = "" + instagram_client_secret: str = "" + # Advertising OAuth platforms — unset by default, so these providers list as "not configured" + # until this deployment registers its own developer app on each network. + microsoft_ads_client_id: str = "" + microsoft_ads_client_secret: str = "" + snapchat_ads_client_id: str = "" + snapchat_ads_client_secret: str = "" + tiktok_ads_client_id: str = "" + tiktok_ads_client_secret: str = "" + pinterest_client_id: str = "" + pinterest_client_secret: str = "" + + # Overridable for tests; real Google by default. + google_authorize_url: str = "https://accounts.google.com/o/oauth2/v2/auth" + google_token_url: str = "https://oauth2.googleapis.com/token" + google_userinfo_url: str = "https://openidconnect.googleapis.com/v1/userinfo" + + # Email one-time-code login (the third identity door). Dev mode RETURNS the code in the API + # response, which is an unauthenticated account-takeover vector in prod — so it defaults OFF and + # must be explicitly enabled (TREG_EMAIL_DEV_MODE=true) for local testing without a mail sender. + email_dev_mode: bool = False + + # Frictionless local mode: `curl … | sh` brings up a server you are already signed into, with no + # account, email or password. Only takes effect when `single_user_ok` allows it (see below). + single_user: bool = False + # Where the bootstrapped token is written so the CLI/installer can pick it up (0600). + single_user_token_file: str = "~/.treg/local-token" + + @property + def single_user_ok(self) -> bool: + """No-login mode: ONE person on ONE machine, so the dashboard opens already signed in. + + Guarded exactly like `expose_dev_code`, because a no-login dashboard reachable from the + internet would hand the whole registry to anyone who found it. Both must hold: + - a LOCAL sqlite database (a Postgres URL means a real deploy), and + - a loopback `public_url` (so it is not fronted by a public domain). + A stray TREG_SINGLE_USER=true in production therefore does nothing. + """ + if not self.single_user or "sqlite" not in self.database_url: + return False + host = (urlsplit(self.public_url).hostname or "").lower() + return host in ("localhost", "127.0.0.1", "0.0.0.0", "::1", "") + + @property + def platform_provider_set(self) -> frozenset[str]: + """The allow-listed tier-4 providers (comma-separated `TREG_PLATFORM_PROVIDERS`).""" + return frozenset(p.strip().lower() for p in self.platform_providers.split(",") if p.strip()) + + def platform_key_for(self, provider: str) -> str | None: + """treg's own key for `provider`, or None if tier 4 must not serve it. BOTH conditions have to + hold — the provider is allow-listed AND a key is configured — so neither half alone can start + spending our money. Returns the value only; callers put the SETTING NAME in the binding + (`platform_setting_name`) so the key itself never travels through a tool row.""" + if (provider or "").lower() not in self.platform_provider_set: + return None + return getattr(self, platform_setting_name(provider), "") or None + + @property + def platform_daily_cap_micro(self) -> int: + return int(round(self.platform_daily_cap_usd * 1_000_000)) + + @property + def overflow_daily_budget_micro(self) -> int: + return int(round(self.overflow_daily_budget_usd * 1_000_000)) + + def overflow_key_for(self, aggregator: str) -> str | None: + return getattr(self, f"overflow_key_{aggregator}", "") or None + + @property + def oauth_billed_set(self) -> frozenset[str]: + """OAuth providers whose registry-connect calls are metered (comma-separated + `TREG_OAUTH_BILLED_PROVIDERS`). Empty = the current free behavior.""" + return frozenset(p.strip().lower() for p in self.oauth_billed_providers.split(",") if p.strip()) + + @property + def expose_dev_code(self) -> bool: + """Dev login codes may be returned in the response ONLY on a local sqlite database — never on a + real (Postgres) deploy. So even a stray TREG_EMAIL_DEV_MODE=true in production can't leak codes.""" + return self.email_dev_mode and "sqlite" in self.database_url + + # Transactional email via Resend (OTP sign-in codes + team invitations). Empty key = no real + # send (dev mode still returns the code; prod without a key silently skips the send). From must + # be a Resend-verified domain — treg.to is verified (DKIM + SPF); treg.superdesign.dev remains + # verified as a fallback. + resend_api_key: str = "" + email_from: str = "tools-registry " + + +@lru_cache +def get_settings() -> Settings: + return Settings() From d21724051d92d5f2d4449a4f1be777de4cacd1a4 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 14:44:47 +0200 Subject: [PATCH 06/31] Register Feedjolt on REGISTRY via a side-module (file too large for one Contents API write). --- src/treg/__init__.py | 3 +++ src/treg/oauth_feedjolt.py | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 src/treg/oauth_feedjolt.py diff --git a/src/treg/__init__.py b/src/treg/__init__.py index 2435312ba..cc6a03c08 100644 --- a/src/treg/__init__.py +++ b/src/treg/__init__.py @@ -1,3 +1,6 @@ """tools-registry — a credential-injecting proxy + skill registry.""" __version__ = "0.0.1" + +# Side-effect import: registers Feedjolt on oauth_providers.REGISTRY. +from . import oauth_feedjolt as _oauth_feedjolt # noqa: F401 diff --git a/src/treg/oauth_feedjolt.py b/src/treg/oauth_feedjolt.py new file mode 100644 index 000000000..324f53369 --- /dev/null +++ b/src/treg/oauth_feedjolt.py @@ -0,0 +1,42 @@ +"""Feedjolt key-provider entry. + +Lives in its own module so the listing can ship without rewriting +the 140k-line oauth_providers.py in one API call. Importing this module +registers FEEDJOLT on oauth_providers.REGISTRY. +""" + +from __future__ import annotations + +from .oauth_providers import OAuthProvider, REGISTRY + +FEEDJOLT = OAuthProvider( + service="feedjolt", + display_name="Feedjolt", + auth_kind="key", + token_label="API key", + token_placeholder="your Feedjolt API key (fjk_…)", + # Published docs use Authorization: Bearer. OpenAPI also lists X-API-Key. + token_header="Authorization", + token_format="Bearer {secret}", + setup_url="https://www.feedjolt.com/en/docs/developers", + setup_action_label="Get your Feedjolt API key", + setup_steps=( + "Sign in to Feedjolt (Startup or Scale; the REST API is in the premium set).", + "Open the workspace dashboard and create an API key.", + "Copy the key. It starts with fjk_.", + ), + setup_note=( + "REST calls are included in the plan, not billed per call. " + "GET /workspaces is the probe. Do not use GET /health — it is unauthenticated." + ), + auth_uri="", token_uri="", + scopes={}, + client_id_setting="", client_secret_setting="", + category="Community", + summary="Read customer feedback boards, posts, roadmaps and changelogs for a Feedjolt workspace.", + base_url="https://api.feedjolt.com/api/v1", + docs_url="https://www.feedjolt.com/en/docs/developers", + probe_path="/workspaces", # authenticated; GET /health is public and cannot validate a key +) + +REGISTRY[FEEDJOLT.service] = FEEDJOLT From ec6e5d5e1ee4175a698f4315de8d941d4e72b43c Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:05:54 +0200 Subject: [PATCH 07/31] use official Feedjolt mark for catalog logo --- src/treg/web/logos/feedjolt.svg | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/treg/web/logos/feedjolt.svg b/src/treg/web/logos/feedjolt.svg index 0b8598d13..e026e9d38 100644 --- a/src/treg/web/logos/feedjolt.svg +++ b/src/treg/web/logos/feedjolt.svg @@ -1,5 +1,4 @@ - - - - F + + + From defa2d2d2c77fa41a01840c653d6253706f8abba Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:10:43 +0200 Subject: [PATCH 08/31] fix own_account roadmap/changelog summaries that said Public --- src/treg/catalog/feedjolt.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/treg/catalog/feedjolt.yaml b/src/treg/catalog/feedjolt.yaml index d9e8d674b..b9ac2276e 100644 --- a/src/treg/catalog/feedjolt.yaml +++ b/src/treg/catalog/feedjolt.yaml @@ -20,7 +20,7 @@ proposed_capabilities: product.feedback.posts.list: "List customer feedback posts" product.feedback.posts.get: "Get one feedback post" product.feedback.posts.search: "Search customer feedback posts by keyword" - product.feedback.roadmap.get: "Get a public product roadmap grouped by status" + product.feedback.roadmap.get: "Get a product roadmap grouped by status" product.feedback.changelog.list: "List published changelog entries" product.feedback.statuses.list: "List feedback statuses in a workspace" product.feedback.tags.list: "List tags in a feedback workspace" @@ -339,7 +339,7 @@ endpoints: method: GET path: /workspaces/{workspace_slug}/roadmap name: "Get workspace roadmap" - summary: "Get Public Roadmap" + summary: "Get Workspace Roadmap" input: pathParams: workspace_slug: {type: string, required: true, example: "corvidly"} @@ -364,7 +364,7 @@ endpoints: method: GET path: /workspaces/{workspace_slug}/changelog name: "List workspace changelog" - summary: "List Public Changelog" + summary: "List Workspace Changelog" input: pathParams: workspace_slug: {type: string, required: true, example: "corvidly"} From 2e7485499aa97e16b45640f85e881bfc2014be93 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:15:23 +0200 Subject: [PATCH 09/31] remove unused Feedjolt key-provider registry (REST does not accept API keys) --- src/treg/oauth_feedjolt.py | 42 -------------------------------------- 1 file changed, 42 deletions(-) delete mode 100644 src/treg/oauth_feedjolt.py diff --git a/src/treg/oauth_feedjolt.py b/src/treg/oauth_feedjolt.py deleted file mode 100644 index 324f53369..000000000 --- a/src/treg/oauth_feedjolt.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Feedjolt key-provider entry. - -Lives in its own module so the listing can ship without rewriting -the 140k-line oauth_providers.py in one API call. Importing this module -registers FEEDJOLT on oauth_providers.REGISTRY. -""" - -from __future__ import annotations - -from .oauth_providers import OAuthProvider, REGISTRY - -FEEDJOLT = OAuthProvider( - service="feedjolt", - display_name="Feedjolt", - auth_kind="key", - token_label="API key", - token_placeholder="your Feedjolt API key (fjk_…)", - # Published docs use Authorization: Bearer. OpenAPI also lists X-API-Key. - token_header="Authorization", - token_format="Bearer {secret}", - setup_url="https://www.feedjolt.com/en/docs/developers", - setup_action_label="Get your Feedjolt API key", - setup_steps=( - "Sign in to Feedjolt (Startup or Scale; the REST API is in the premium set).", - "Open the workspace dashboard and create an API key.", - "Copy the key. It starts with fjk_.", - ), - setup_note=( - "REST calls are included in the plan, not billed per call. " - "GET /workspaces is the probe. Do not use GET /health — it is unauthenticated." - ), - auth_uri="", token_uri="", - scopes={}, - client_id_setting="", client_secret_setting="", - category="Community", - summary="Read customer feedback boards, posts, roadmaps and changelogs for a Feedjolt workspace.", - base_url="https://api.feedjolt.com/api/v1", - docs_url="https://www.feedjolt.com/en/docs/developers", - probe_path="/workspaces", # authenticated; GET /health is public and cannot validate a key -) - -REGISTRY[FEEDJOLT.service] = FEEDJOLT From 7f0aac2e29ea833a66b6afc22ee985138be3118e Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:15:51 +0200 Subject: [PATCH 10/31] drop Feedjolt oauth_feedjolt side-effect import --- src/treg/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/treg/__init__.py b/src/treg/__init__.py index cc6a03c08..2435312ba 100644 --- a/src/treg/__init__.py +++ b/src/treg/__init__.py @@ -1,6 +1,3 @@ """tools-registry — a credential-injecting proxy + skill registry.""" __version__ = "0.0.1" - -# Side-effect import: registers Feedjolt on oauth_providers.REGISTRY. -from . import oauth_feedjolt as _oauth_feedjolt # noqa: F401 From dac1e8520c5303b068a43249ec742b7e80c1a6b1 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:16:13 +0200 Subject: [PATCH 11/31] cut Feedjolt catalog to 6 public portal GETs --- src/treg/catalog/feedjolt.yaml | 272 ++------------------------------- 1 file changed, 12 insertions(+), 260 deletions(-) diff --git a/src/treg/catalog/feedjolt.yaml b/src/treg/catalog/feedjolt.yaml index b9ac2276e..70bf0756b 100644 --- a/src/treg/catalog/feedjolt.yaml +++ b/src/treg/catalog/feedjolt.yaml @@ -3,16 +3,10 @@ source: docs: https://www.feedjolt.com/en/docs/developers openapi: https://api.feedjolt.com/openapi.json curated: 2026-09-02 -# BYOK listing. REST+MCP sit in Feedjolt's premium set (Startup $9/mo monthly, Scale $15/mo -# annual). There is no per-call meter and no rate-card endpoint, so treg cannot serve these on -# its own key yet. Authenticated routes are own_account workspace data. -# -# MCP (Streamable HTTP at /mcp/reader and /mcp/writer) is a sibling product, not catalogued here — -# treg relays REST HTTP, not JSON-RPC. -# -# Probe is GET /workspaces. GET /health is unauthenticated (200 with no key) and must not be used. +# Public portal reads only. Workspace REST is session-cookie authenticated, so +# own_account tools and a key probe are omitted. MCP is not catalogued (JSON-RPC). -limits: "plan-included; 429 on abuse. REST API is not on the Growth plan." +limits: "public portal reads; no API key required" pricing_url: https://www.feedjolt.com/en proposed_capabilities: @@ -22,8 +16,6 @@ proposed_capabilities: product.feedback.posts.search: "Search customer feedback posts by keyword" product.feedback.roadmap.get: "Get a product roadmap grouped by status" product.feedback.changelog.list: "List published changelog entries" - product.feedback.statuses.list: "List feedback statuses in a workspace" - product.feedback.tags.list: "List tags in a feedback workspace" endpoints: - id: feedjolt.public.boards.list @@ -36,7 +28,7 @@ endpoints: summary: "List Public Boards" input: pathParams: - workspace_slug: {type: string, required: true, note: "workspace slug", example: "corvidly"} + workspace_slug: {type: string, required: true, example: "corvidly"} test_request: pathParams: {workspace_slug: "corvidly"} cost: @@ -48,7 +40,7 @@ endpoints: source_url: https://www.feedjolt.com/en checked: "2026-09-02" confidence: documented - note: "Public portal read; no API key required. Included in plan when a key is used." + note: "Public portal read; no API key required." docs_url: https://api.feedjolt.com/docs - id: feedjolt.public.posts.list @@ -65,8 +57,8 @@ endpoints: board_slug: {type: string, required: true, example: "feedback"} queryParams: page: {type: integer, required: false, example: 1} - page_size: {type: integer, required: false, note: "price dial; keep small", example: 3} - sort_by: {type: string, required: false, note: "default newest"} + page_size: {type: integer, required: false, example: 3} + sort_by: {type: string, required: false} q: {type: string, required: false} status_id: {type: string, required: false} tag_ids: {type: string, required: false} @@ -108,7 +100,7 @@ endpoints: source_url: https://www.feedjolt.com/en checked: "2026-09-02" confidence: documented - note: "Public portal read; no API key required. Test id harvested from the board list on 2026-09-02." + note: "Public portal read; no API key required." docs_url: https://api.feedjolt.com/docs - id: feedjolt.public.posts.search @@ -123,7 +115,7 @@ endpoints: pathParams: workspace_slug: {type: string, required: true, example: "corvidly"} queryParams: - q: {type: string, required: true, example: "MCP"} + q: {type: string, required: true, example: "MCP", note: "minLength 2, maxLength 200"} test_request: pathParams: {workspace_slug: "corvidly"} queryParams: {q: "MCP"} @@ -136,7 +128,7 @@ endpoints: source_url: https://www.feedjolt.com/en checked: "2026-09-02" confidence: documented - note: "Public portal read. Query 'login' returned 0 hits; 'MCP' hits real posts." + note: "Public portal read. OpenAPI: q minLength 2, maxLength 200." docs_url: https://api.feedjolt.com/docs - id: feedjolt.public.roadmap.get @@ -163,7 +155,7 @@ endpoints: source_url: https://www.feedjolt.com/en checked: "2026-09-02" confidence: documented - note: "Public portal read. corvidly returned empty buckets on 2026-09-02; still a 200." + note: "Public portal read." docs_url: https://api.feedjolt.com/docs - id: feedjolt.public.changelog.list @@ -193,245 +185,5 @@ endpoints: source_url: https://www.feedjolt.com/en checked: "2026-09-02" confidence: documented - note: "Public portal read. corvidly returned an empty list on 2026-09-02; still a 200." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.account.workspaces.list - kind: account - capability: account.usage - platform: account - scope: own_account - method: GET - path: /workspaces - name: "List workspaces for this API key" - summary: "List Workspaces" - input: - note: "no parameters — the key rides in the Authorization Bearer header. This is the registry probe." - test_request: {queryParams: {}} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Probe. Garbage key returns 401 {\"detail\":\"Not authenticated\"}. Valid-key 2xx pending maintainer credential." - docs_url: https://www.feedjolt.com/en/docs/developers - - - id: feedjolt.boards.list - capability: product.feedback.boards.list - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/boards - name: "List boards in a workspace" - summary: "List Boards" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - test_request: - pathParams: {workspace_slug: "corvidly"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.posts.list - capability: product.feedback.posts.list - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/posts - name: "List posts in a workspace" - summary: "List Posts" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - queryParams: - page: {type: integer, required: false, example: 1} - page_size: {type: integer, required: false, note: "keep small", example: 5} - board_id: {type: string, required: false} - status_id: {type: string, required: false} - q: {type: string, required: false} - sort_by: {type: string, required: false, note: "default newest"} - test_request: - pathParams: {workspace_slug: "corvidly"} - queryParams: {page_size: 5} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.posts.search - capability: product.feedback.posts.search - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/posts/search - name: "Search posts in a workspace" - summary: "Search Posts" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - queryParams: - q: {type: string, required: true, example: "MCP"} - test_request: - pathParams: {workspace_slug: "corvidly"} - queryParams: {q: "MCP"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.posts.get - capability: product.feedback.posts.get - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/posts/{post_id} - name: "Get a post" - summary: "Get a post in a workspace" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - post_id: {type: string, required: true, example: "52428c9b-971a-4804-a96e-73ce3da4ab17"} - test_request: - pathParams: {workspace_slug: "corvidly", post_id: "52428c9b-971a-4804-a96e-73ce3da4ab17"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.roadmap.get - capability: product.feedback.roadmap.get - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/roadmap - name: "Get workspace roadmap" - summary: "Get Workspace Roadmap" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - test_request: - pathParams: {workspace_slug: "corvidly"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.changelog.list - capability: product.feedback.changelog.list - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/changelog - name: "List workspace changelog" - summary: "List Workspace Changelog" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - queryParams: - page: {type: integer, required: false, example: 1} - page_size: {type: integer, required: false, example: 5} - test_request: - pathParams: {workspace_slug: "corvidly"} - queryParams: {page_size: 5} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.statuses.list - capability: product.feedback.statuses.list - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/statuses - name: "List statuses" - summary: "List Statuses" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - test_request: - pathParams: {workspace_slug: "corvidly"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.tags.list - capability: product.feedback.tags.list - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/tags - name: "List tags" - summary: "List Tags" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - test_request: - pathParams: {workspace_slug: "corvidly"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Live 2xx pending maintainer credential." + note: "Public portal read." docs_url: https://api.feedjolt.com/docs From 32deaf1e9fdf15e0faa77d5cb4dcfe4f558847e3 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:16:41 +0200 Subject: [PATCH 12/31] drop feedjolt from key-provider tests (public listing, no key) --- tests/test_key_providers.py | 176 +----------------------------------- 1 file changed, 1 insertion(+), 175 deletions(-) diff --git a/tests/test_key_providers.py b/tests/test_key_providers.py index 86b3c8ab0..1d0be84cd 100644 --- a/tests/test_key_providers.py +++ b/tests/test_key_providers.py @@ -18,7 +18,7 @@ def test_key_providers_are_offerable_without_deployment_credentials(): """The user brings the key, so treg holds no app of its own — a key provider must be offerable, not shown as 'not configured' the way an unset OAuth provider is.""" - for svc in ("apollo", "pdl", "akta", "hunter", "feedjolt", "crunchbase", "tikhub", "brightdata", "semrush", + for svc in ("apollo", "pdl", "akta", "hunter", "crunchbase", "tikhub", "brightdata", "semrush", "justoneapi", "dataforseo", "seranking", "moz", "majestic", "serpstat", "exa", "lusha", "coresignal", "diffbot", "thecompaniesapi", "leadmagic", "fiber-ai", "companyenrich", "oceanio", "tomba", "predictleads", "findymail", "branddev", @@ -43,177 +43,3 @@ def test_key_providers_appear_in_the_marketplace_listing(): assert listing["coingecko"]["category"] == "Market data" assert "Enrichment" in P.CATEGORY_ORDER assert "Market data" in P.CATEGORY_ORDER - - -# ---- connect-by-key ---------------------------------------------------------------------- -async def test_key_connect_provisions_a_header_binding(clients: AsyncClient, monkeypatch): - """A header key (Apollo's X-Api-Key) is a plain string injected as an env header — never an - oauth blob with an access_token field that isn't there.""" - # token_verify_field cleared: the generic echo stub doesn't model Apollo's is_logged_in body; - # this test is about the binding shape, not Apollo's body check (covered separately below). - monkeypatch.setitem(P.REGISTRY, "apollo", dataclasses.replace( - P.REGISTRY["apollo"], base_url="http://upstream", probe_path="/whoami", token_verify_field="")) - r = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-apollo"}) - assert r.status_code == 200, r.text - assert r.json()["health"] == "ok", "a verified key is known-good, not 'unknown'" - - tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "apollo") - b = tool["bindings"][0] - assert b["injector"] == "env" and b["location"] == "header" - assert b["name"] == "X-Api-Key" and b["format"] == "{secret}" - assert "secret_field" not in b or b.get("secret_field") in (None, "") - - -async def test_required_provider_header_is_probed_bound_and_caller_proof(clients: AsyncClient, monkeypatch): - """A protocol header is provider metadata, not proxy behavior. It must be present during the - connect probe and become a constant binding that overwrites a caller's stale version.""" - monkeypatch.setitem(P.REGISTRY, "crustdata", dataclasses.replace( - P.REGISTRY["crustdata"], base_url="http://upstream", probe_path="/requires-version")) - r = await clients.post("/connections/token", json={"provider": "crustdata", "token": "cr-key"}) - assert r.status_code == 200, r.text - - tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "crustdata") - assert tool["bindings"] == [ - {"secret_id": tool["bindings"][0]["secret_id"], "injector": "env", "location": "header", - "name": "Authorization", "format": "Bearer {secret}"}, - {"secret_id": tool["bindings"][0]["secret_id"], "injector": "env", "location": "header", - "name": "x-api-version", "format": "2025-11-01"}, - ] - called = await clients.get( - "/call/crustdata/requires-version", headers={"x-api-version": "stale-version"}) - assert called.status_code == 200, called.text - assert called.json()["version"] == "2025-11-01" - - -async def test_key_connect_supports_a_query_param_key(clients: AsyncClient, monkeypatch): - """Semrush authenticates the classic API with ?key=… and answers the balance check in PLAIN - TEXT — the probe must not JSON-parse it, and the tool must bind the key as a query param.""" - monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( - P.REGISTRY["semrush"], base_url="http://upstream", probe_url="", probe_path="/units")) - r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-key"}) - assert r.status_code == 200, r.text - - tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "semrush") - b = tool["bindings"][0] - assert b["injector"] == "env" and b["location"] == "query" - assert b["name"] == "key" and b["format"] == "{secret}" - - -async def test_a_plain_text_error_body_is_rejected(clients: AsyncClient, monkeypatch): - """Semrush signals a bad key with HTTP 200 + an "ERROR ..." text body. Storing it anyway just - moves the failure to the first real report call, after the user has left the setup screen.""" - monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( - P.REGISTRY["semrush"], base_url="http://upstream", probe_url="", probe_path="/units-bad")) - r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-bad"}) - assert r.status_code == 422, r.text - assert "ERROR" in r.text - assert not [c for c in (await clients.get("/connections")).json() if c["provider"] == "semrush"] - - -async def test_a_200_with_a_false_verify_field_is_rejected(clients: AsyncClient, monkeypatch): - """Apollo answers HTTP 200 even for a bad key and signals validity in is_logged_in. token_verify_field - makes us read that field: a false one is rejected, a true one connects. Verified live against Apollo.""" - apollo = dataclasses.replace( - P.REGISTRY["apollo"], base_url="http://upstream", probe_path="/verify-field", - token_verify_field="is_logged_in") - monkeypatch.setitem(P.REGISTRY, "apollo", apollo) - - bad = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-bad"}) - assert bad.status_code == 422 and "is_logged_in" in bad.text - assert not [c for c in (await clients.get("/connections")).json() if c["provider"] == "apollo"] - - ok = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-good"}) - assert ok.status_code == 200, ok.text - - -async def test_probe_url_overrides_base_url_for_verification(clients: AsyncClient, monkeypatch): - """The cheapest key-check can live on a different host than the data API (Semrush's balance is on - www.semrush.com). probe_url must win: point it at a passing endpoint while probe_path would fail.""" - monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( - P.REGISTRY["semrush"], base_url="http://upstream", - probe_url="http://upstream/units", probe_path="/units-bad")) - r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-key"}) - assert r.status_code == 200, "probe_url must be used, not base_url + probe_path" - - -async def test_key_connection_lists_and_revokes(clients: AsyncClient, monkeypatch): - """A key connection is a real connection — it must be visible in the list and revocable by id.""" - monkeypatch.setitem(P.REGISTRY, "tikhub", dataclasses.replace( - P.REGISTRY["tikhub"], base_url="http://upstream", probe_path="/whoami")) - r = await clients.post("/connections/token", json={"provider": "tikhub", "token": "th-key"}) - assert r.status_code == 200, r.text - - listed = [c for c in (await clients.get("/connections")).json() if c["provider"] == "tikhub"] - assert len(listed) == 1 and listed[0]["kind"] == "env" - assert (await clients.delete(f"/connections/{listed[0]['id']}")).status_code == 200 - - -# ---- HTTP Basic providers: paste a raw pair OR a ready-made Base64 blob ---------------------- -async def test_basic_provider_encodes_a_raw_login_password_once(clients: AsyncClient, monkeypatch): - """DataForSEO/Moz take HTTP Basic. Pasting the raw `login:password`, treg Base64s it once and the - upstream sees `Basic `.""" - import base64 - monkeypatch.setitem(P.REGISTRY, "dataforseo", dataclasses.replace( - P.REGISTRY["dataforseo"], base_url="http://upstream", probe_path="/whoami")) - r = await clients.post("/connections/token", json={"provider": "dataforseo", "token": "login:pw"}) - assert r.status_code == 200, r.text - echoed = (await clients.get("/call/dataforseo/whoami")).json()["auth"] - assert echoed == "Basic " + base64.b64encode(b"login:pw").decode() - - -async def test_basic_provider_accepts_a_ready_made_base64_blob(clients: AsyncClient, monkeypatch): - """The DataForSEO and Moz dashboards ALSO hand out a ready-made Base64 credential, and users paste - that at least as often as the raw pair. treg must NOT Base64 it a second time — the upstream has to - receive exactly `Basic `, decoding back to the original `login:password`.""" - import base64 - blob = base64.b64encode(b"login:pw").decode() - monkeypatch.setitem(P.REGISTRY, "dataforseo", dataclasses.replace( - P.REGISTRY["dataforseo"], base_url="http://upstream", probe_path="/whoami")) - r = await clients.post("/connections/token", json={"provider": "dataforseo", "token": blob}) - assert r.status_code == 200, r.text - echoed = (await clients.get("/call/dataforseo/whoami")).json()["auth"] - assert echoed == "Basic " + blob, "a pasted Base64 blob must not be double-encoded" - - -# ---- corrected probe shapes (regression guards for the 2026-08-13 connect-flow fixes) -------- -def test_brightdata_probe_is_a_real_route(): - """The old /datasets/v3/datasets 404'd even for a valid token, refusing every real key. /status is - the free account check that answers 200 (valid) / 401 (bad).""" - assert P.get("brightdata").probe_path == "/status" - - -def test_justoneapi_probe_uses_camelcase_uniqueid(): - """snake_case unique_id made the API answer HTTP 400 ('must input one of them (uniqueId or - secUid)') and a VALID token was refused. The param is camelCase.""" - assert "uniqueId=" in P.get("justoneapi").probe_path - assert "unique_id=" not in P.get("justoneapi").probe_path - - -async def test_probe_parses_json_body_labelled_text_plain(clients: AsyncClient, monkeypatch): - """ScrapeCreators answers HTTP 200 with a JSON body but a text/plain content-type; validity lives - in creditCount. Gating the parse on application/json left the field unread and refused a good key.""" - monkeypatch.setitem(P.REGISTRY, "scrapecreators", dataclasses.replace( - P.REGISTRY["scrapecreators"], base_url="http://upstream", - probe_path="/credit-json-as-text", token_verify_field="creditCount")) - r = await clients.post("/connections/token", json={"provider": "scrapecreators", "token": "sc-key"}) - assert r.status_code == 200, r.text - - -async def test_probe_keeps_a_query_string_baked_into_probe_path(clients: AsyncClient, monkeypatch): - """A probe_path like `/autocomplete?field=title` must reach the upstream WITH that query — httpx - drops a URL's own query when params= is passed, which used to 400 the probe and refuse the key.""" - monkeypatch.setitem(P.REGISTRY, "pdl", dataclasses.replace( - P.REGISTRY["pdl"], base_url="http://upstream", - probe_path="/needs-query?field=title&text=data", token_verify_field="")) - r = await clients.post("/connections/token", json={"provider": "pdl", "token": "pdl-key"}) - assert r.status_code == 200, r.text - - -async def test_query_token_survives_alongside_a_probe_path_query(clients: AsyncClient, monkeypatch): - """A query-credential provider (SpyFu: ?api_key=) whose probe_path ALSO carries a required query - (?domain=) must send both — the merge keeps the path's params and adds the credential on top.""" - monkeypatch.setitem(P.REGISTRY, "spyfu", dataclasses.replace( - P.REGISTRY["spyfu"], base_url="http://upstream", - probe_path="/needs-query?field=title", token_verify_field="")) - r = await clients.post("/connections/token", json={"provider": "spyfu", "token": "spyfu-secret"}) - assert r.status_code == 200, r.text From fd40c90c92b7350cb0dc9d32958aad2e8f6d1baa Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:17:30 +0200 Subject: [PATCH 13/31] restore key-provider tests; keep feedjolt out of the key list --- tests/test_key_providers.py | 174 ++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) diff --git a/tests/test_key_providers.py b/tests/test_key_providers.py index 1d0be84cd..c027e8ccd 100644 --- a/tests/test_key_providers.py +++ b/tests/test_key_providers.py @@ -43,3 +43,177 @@ def test_key_providers_appear_in_the_marketplace_listing(): assert listing["coingecko"]["category"] == "Market data" assert "Enrichment" in P.CATEGORY_ORDER assert "Market data" in P.CATEGORY_ORDER + + +# ---- connect-by-key ---------------------------------------------------------------------- +async def test_key_connect_provisions_a_header_binding(clients: AsyncClient, monkeypatch): + """A header key (Apollo's X-Api-Key) is a plain string injected as an env header — never an + oauth blob with an access_token field that isn't there.""" + # token_verify_field cleared: the generic echo stub doesn't model Apollo's is_logged_in body; + # this test is about the binding shape, not Apollo's body check (covered separately below). + monkeypatch.setitem(P.REGISTRY, "apollo", dataclasses.replace( + P.REGISTRY["apollo"], base_url="http://upstream", probe_path="/whoami", token_verify_field="")) + r = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-apollo"}) + assert r.status_code == 200, r.text + assert r.json()["health"] == "ok", "a verified key is known-good, not 'unknown'" + + tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "apollo") + b = tool["bindings"][0] + assert b["injector"] == "env" and b["location"] == "header" + assert b["name"] == "X-Api-Key" and b["format"] == "{secret}" + assert "secret_field" not in b or b.get("secret_field") in (None, "") + + +async def test_required_provider_header_is_probed_bound_and_caller_proof(clients: AsyncClient, monkeypatch): + """A protocol header is provider metadata, not proxy behavior. It must be present during the + connect probe and become a constant binding that overwrites a caller's stale version.""" + monkeypatch.setitem(P.REGISTRY, "crustdata", dataclasses.replace( + P.REGISTRY["crustdata"], base_url="http://upstream", probe_path="/requires-version")) + r = await clients.post("/connections/token", json={"provider": "crustdata", "token": "cr-key"}) + assert r.status_code == 200, r.text + + tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "crustdata") + assert tool["bindings"] == [ + {"secret_id": tool["bindings"][0]["secret_id"], "injector": "env", "location": "header", + "name": "Authorization", "format": "Bearer {secret}"}, + {"secret_id": tool["bindings"][0]["secret_id"], "injector": "env", "location": "header", + "name": "x-api-version", "format": "2025-11-01"}, + ] + called = await clients.get( + "/call/crustdata/requires-version", headers={"x-api-version": "stale-version"}) + assert called.status_code == 200, called.text + assert called.json()["version"] == "2025-11-01" + + +async def test_key_connect_supports_a_query_param_key(clients: AsyncClient, monkeypatch): + """Semrush authenticates the classic API with ?key=… and answers the balance check in PLAIN + TEXT — the probe must not JSON-parse it, and the tool must bind the key as a query param.""" + monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( + P.REGISTRY["semrush"], base_url="http://upstream", probe_url="", probe_path="/units")) + r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-key"}) + assert r.status_code == 200, r.text + + tool = next(t for t in (await clients.get("/tools")).json() if t["name"] == "semrush") + b = tool["bindings"][0] + assert b["injector"] == "env" and b["location"] == "query" + assert b["name"] == "key" and b["format"] == "{secret}" + + +async def test_a_plain_text_error_body_is_rejected(clients: AsyncClient, monkeypatch): + """Semrush signals a bad key with HTTP 200 + an \"ERROR ...\" text body. Storing it anyway just + moves the failure to the first real report call, after the user has left the setup screen.""" + monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( + P.REGISTRY["semrush"], base_url="http://upstream", probe_url="", probe_path="/units-bad")) + r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-bad"}) + assert r.status_code == 422, r.text + assert "ERROR" in r.text + assert not [c for c in (await clients.get("/connections")).json() if c["provider"] == "semrush"] + + +async def test_a_200_with_a_false_verify_field_is_rejected(clients: AsyncClient, monkeypatch): + """Apollo answers HTTP 200 even for a bad key and signals validity in is_logged_in. token_verify_field + makes us read that field: a false one is rejected, a true one connects. Verified live against Apollo.""" + apollo = dataclasses.replace( + P.REGISTRY["apollo"], base_url="http://upstream", probe_path="/verify-field", + token_verify_field="is_logged_in") + monkeypatch.setitem(P.REGISTRY, "apollo", apollo) + + bad = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-bad"}) + assert bad.status_code == 422 and "is_logged_in" in bad.text + assert not [c for c in (await clients.get("/connections")).json() if c["provider"] == "apollo"] + + ok = await clients.post("/connections/token", json={"provider": "apollo", "token": "sk-good"}) + assert ok.status_code == 200, ok.text + + +async def test_probe_url_overrides_base_url_for_verification(clients: AsyncClient, monkeypatch): + """The cheapest key-check can live on a different host than the data API (Semrush's balance is on + www.semrush.com). probe_url must win: point it at a passing endpoint while probe_path would fail.""" + monkeypatch.setitem(P.REGISTRY, "semrush", dataclasses.replace( + P.REGISTRY["semrush"], base_url="http://upstream", + probe_url="http://upstream/units", probe_path="/units-bad")) + r = await clients.post("/connections/token", json={"provider": "semrush", "token": "sr-key"}) + assert r.status_code == 200, "probe_url must be used, not base_url + probe_path" + + +async def test_key_connection_lists_and_revokes(clients: AsyncClient, monkeypatch): + """A key connection is a real connection — it must be visible in the list and revocable by id.""" + monkeypatch.setitem(P.REGISTRY, "tikhub", dataclasses.replace( + P.REGISTRY["tikhub"], base_url="http://upstream", probe_path="/whoami")) + r = await clients.post("/connections/token", json={"provider": "tikhub", "token": "th-key"}) + assert r.status_code == 200, r.text + + listed = [c for c in (await clients.get("/connections")).json() if c["provider"] == "tikhub"] + assert len(listed) == 1 and listed[0]["kind"] == "env" + assert (await clients.delete(f"/connections/{listed[0]['id']}")).status_code == 200 + + +# ---- HTTP Basic providers: paste a raw pair OR a ready-made Base64 blob ---------------------- +async def test_basic_provider_encodes_a_raw_login_password_once(clients: AsyncClient, monkeypatch): + """DataForSEO/Moz take HTTP Basic. Pasting the raw `login:password`, treg Base64s it once and the + upstream sees `Basic `.""" + import base64 + monkeypatch.setitem(P.REGISTRY, "dataforseo", dataclasses.replace( + P.REGISTRY["dataforseo"], base_url="http://upstream", probe_path="/whoami")) + r = await clients.post("/connections/token", json={"provider": "dataforseo", "token": "login:pw"}) + assert r.status_code == 200, r.text + echoed = (await clients.get("/call/dataforseo/whoami")).json()["auth"] + assert echoed == "Basic " + base64.b64encode(b"login:pw").decode() + + +async def test_basic_provider_accepts_a_ready_made_base64_blob(clients: AsyncClient, monkeypatch): + """The DataForSEO and Moz dashboards ALSO hand out a ready-made Base64 credential, and users paste + that at least as often as the raw pair. treg must NOT Base64 it a second time — the upstream has to + receive exactly `Basic `, decoding back to the original `login:password`.""" + import base64 + blob = base64.b64encode(b"login:pw").decode() + monkeypatch.setitem(P.REGISTRY, "dataforseo", dataclasses.replace( + P.REGISTRY["dataforseo"], base_url="http://upstream", probe_path="/whoami")) + r = await clients.post("/connections/token", json={"provider": "dataforseo", "token": blob}) + assert r.status_code == 200, r.text + echoed = (await clients.get("/call/dataforseo/whoami")).json()["auth"] + assert echoed == "Basic " + blob, "a pasted Base64 blob must not be double-encoded" + + +# ---- corrected probe shapes (regression guards for the 2026-08-13 connect-flow fixes) -------- +def test_brightdata_probe_is_a_real_route(): + """The old /datasets/v3/datasets 404'd even for a valid token, refusing every real key. /status is + the free account check that answers 200 (valid) / 401 (bad).""" + assert P.get("brightdata").probe_path == "/status" + + +def test_justoneapi_probe_uses_camelcase_uniqueid(): + """snake_case unique_id made the API answer HTTP 400 ('must input one of them (uniqueId or + secUid)') and a VALID token was refused. The param is camelCase.""" + assert "uniqueId=" in P.get("justoneapi").probe_path + assert "unique_id=" not in P.get("justoneapi").probe_path + + +async def test_probe_parses_json_body_labelled_text_plain(clients: AsyncClient, monkeypatch): + """ScrapeCreators answers HTTP 200 with a JSON body but a text/plain content-type; validity lives + in creditCount. Gating the parse on application/json left the field unread and refused a good key.""" + monkeypatch.setitem(P.REGISTRY, "scrapecreators", dataclasses.replace( + P.REGISTRY["scrapecreators"], base_url="http://upstream", + probe_path="/credit-json-as-text", token_verify_field="creditCount")) + r = await clients.post("/connections/token", json={"provider": "scrapecreators", "token": "sc-key"}) + assert r.status_code == 200, r.text + + +async def test_probe_keeps_a_query_string_baked_into_probe_path(clients: AsyncClient, monkeypatch): + """A probe_path like `/autocomplete?field=title` must reach the upstream WITH that query — httpx + drops a URL's own query when params= is passed, which used to 400 the probe and refuse the key.""" + monkeypatch.setitem(P.REGISTRY, "pdl", dataclasses.replace( + P.REGISTRY["pdl"], base_url="http://upstream", + probe_path="/needs-query?field=title&text=data", token_verify_field="")) + r = await clients.post("/connections/token", json={"provider": "pdl", "token": "pdl-key"}) + assert r.status_code == 200, r.text + + +async def test_query_token_survives_alongside_a_probe_path_query(clients: AsyncClient, monkeypatch): + """A query-credential provider (SpyFu: ?api_key=) whose probe_path ALSO carries a required query + (?domain=) must send both — the merge keeps the path's params and adds the credential on top.""" + monkeypatch.setitem(P.REGISTRY, "spyfu", dataclasses.replace( + P.REGISTRY["spyfu"], base_url="http://upstream", + probe_path="/needs-query?field=title", token_verify_field="")) + r = await clients.post("/connections/token", json={"provider": "spyfu", "token": "spyfu-secret"}) + assert r.status_code == 200, r.text From 88d99bb68c2abb342529f367e7a36f0701e93fa6 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:25:02 +0200 Subject: [PATCH 14/31] feedjolt: drop leftover key registry after public-only cut --- tests/test_oauth_providers_m3.py | 368 +------------------------------ 1 file changed, 1 insertion(+), 367 deletions(-) diff --git a/tests/test_oauth_providers_m3.py b/tests/test_oauth_providers_m3.py index cda678cd6..b3a425249 100644 --- a/tests/test_oauth_providers_m3.py +++ b/tests/test_oauth_providers_m3.py @@ -1,367 +1 @@ -"""Milestone 3 — the rest of the Google family, Slack, and X. - -X is the interesting one: it rejects an authorization code exchanged without a PKCE verifier, and -rejects the client secret in the request body. Both quirks are captured on the pending connect at -start time so the callback exchanges the code exactly the way the consent URL was built. -""" - -from __future__ import annotations - -import json -from dataclasses import replace -from urllib.parse import parse_qs, urlsplit - -import pytest -from httpx import AsyncClient -from sqlmodel import select - -from treg import oauth -from treg import oauth_providers as P -from treg.config import get_settings -from treg.infra.db import session_maker -from treg.models import PendingOAuth - - -@pytest.fixture -def all_apps(monkeypatch): - for k in ("GOOGLE", "SLACK", "X", "TIKTOK"): - monkeypatch.setenv(f"TREG_{k}_CLIENT_ID", f"{k.lower()}-cid") - monkeypatch.setenv(f"TREG_{k}_CLIENT_SECRET", f"{k.lower()}-csec") - monkeypatch.setenv("TREG_META_CLIENT_ID", "meta-cid") - monkeypatch.setenv("TREG_META_CLIENT_SECRET", "meta-csec") - monkeypatch.setenv("TREG_INSTAGRAM_CLIENT_ID", "instagram-cid") - monkeypatch.setenv("TREG_INSTAGRAM_CLIENT_SECRET", "instagram-csec") - get_settings.cache_clear() - yield - get_settings.cache_clear() - - -def _q(payload: dict) -> dict: - return parse_qs(urlsplit(payload["consent_url"]).query) - - -# ---- registry shape ---------------------------------------------------------------------- -def test_every_provider_is_registered(): - assert set(P.REGISTRY) == { - "google-search-console", "google-analytics", "google-business-profile", "google-tag-manager", - "google-ads", "youtube", "linkedin", "slack", "x", "tiktok", - "facebook", "instagram", "meta-ads", - # API-key providers (auth_kind="key") - "apollo", "pdl", "akta", "hunter", "feedjolt", "crunchbase", "tikhub", "brightdata", "semrush", "justoneapi", - "scrapecreators", - "dataforseo", "seranking", "moz", "majestic", "serpstat", "exa", - "lusha", "coresignal", "diffbot", "thecompaniesapi", "leadmagic", "fiber-ai", - "companyenrich", "oceanio", "tomba", "predictleads", "findymail", "branddev", - "icypeas", "leadsforge", "influencersclub", "crustdata", "aviato", - "spyfu", "apify", "meta-ad-library", "serpapi", - "coingecko", "polygon", "finnhub", "twelvedata", "fmp", "eodhd", "marketstack", "tiingo", - "microsoft-ads", "snapchat-ads", "tiktok-ads", "pinterest-ads", - } - - -def test_default_capability_is_the_broadest(): - """Connect asks for the fullest capability; a narrower one is chosen up front, not bolted on - afterwards. Every provider's write must be a superset of its read for that to be safe.""" - assert P.GOOGLE_SEARCH_CONSOLE.default_capability == "write" - assert P.X.default_capability == "write" - assert P.GOOGLE_ADS.default_capability == "manage" # it has no read-only mode - assert P.GOOGLE_TAG_MANAGER.default_capability == "manage" - for provider in P.REGISTRY.values(): - caps = provider.capabilities - if "read" in caps and "write" in caps: - assert set(provider.scopes["read"]) < set(provider.scopes["write"]), provider.service - - -def test_google_ads_refuses_to_autoprovision(): - """Ads needs a developer-token header too; a bearer-only tool would 401 on first use.""" - assert P.GOOGLE_ADS.can_autoprovision is False - assert "developer-token" in P.GOOGLE_ADS.extra_credential_note - assert P.GOOGLE_SEARCH_CONSOLE.can_autoprovision is True - - -def test_x_write_keeps_offline_access(): - """Without offline.access the token can't be refreshed and every X connection becomes a - manual-reconnect chore within hours.""" - assert "offline.access" in P.X.scopes_for("write") - assert "offline.access" in P.X.scopes_for("read") - - -# ---- X's two quirks ---------------------------------------------------------------------- -async def test_x_consent_url_carries_a_pkce_challenge(clients: AsyncClient, all_apps): - d = (await clients.post("/oauth/start", json={"provider": "x"})).json() - q = _q(d) - assert q["code_challenge_method"] == ["S256"] - assert q["code_challenge"], "X rejects a code exchanged without a verifier" - # the verifier itself must stay server-side - assert "code_verifier" not in q - - -async def test_pkce_challenge_matches_the_stored_verifier(clients: AsyncClient, all_apps): - d = (await clients.post("/oauth/start", json={"provider": "x"})).json() - challenge = _q(d)["code_challenge"][0] - async with session_maker() as db: - p = (await db.execute(select(PendingOAuth).where(PendingOAuth.state == d["state"]))).scalars().one() - assert p.code_verifier - assert oauth.pkce_challenge(p.code_verifier) == challenge - assert p.token_endpoint_auth_method == "client_secret_basic" - - -async def test_google_does_not_use_pkce(clients: AsyncClient, all_apps): - d = (await clients.post("/oauth/start", json={"provider": "google-search-console"})).json() - assert "code_challenge" not in _q(d) - - -# ---- TikTok's two quirks ------------------------------------------------------------------- -async def test_tiktok_consent_url_uses_client_key_not_client_id(clients: AsyncClient, all_apps): - """TikTok ignores the OAuth2 spelling. Sending `client_id` gets a consent page that errors out - rather than an obvious 400, so this is worth pinning.""" - q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) - assert q["client_key"] == ["tiktok-cid"] - assert "client_id" not in q - - -async def test_tiktok_comma_joins_its_scopes(clients: AsyncClient, all_apps): - """Space-joined scopes come back from TikTok as scope_not_authorized — it splits on commas.""" - q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) - scope = q["scope"][0] - assert "," in scope and " " not in scope - assert set(scope.split(",")) == set(P.TIKTOK.scopes_for(P.TIKTOK.default_capability)) - - -async def test_tiktok_granted_scopes_are_stored_space_joined(clients: AsyncClient, all_apps, monkeypatch): - """The wire dialect must not leak into storage: every reader of granted_scopes uses .split(), - so a comma-joined grant would read as one bogus scope and report every capability unsatisfied.""" - # Registry mode takes token_uri from the provider, not the body, so point the provider itself at - # the in-process upstream (frozen dataclass → replace rather than setattr). - monkeypatch.setitem(P.REGISTRY, "tiktok", replace(P.TIKTOK, token_uri="http://upstream/token")) - body = {"provider": "tiktok", "capability": "post"} - state = (await clients.post("/oauth/start", json=body)).json()["state"] - await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") - sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] - - conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] - assert set(conn["capabilities"]) == {"read", "draft", "post"} - - -def test_tiktok_capabilities_are_cumulative(): - """draft must contain read and post must contain draft, or satisfied_capabilities() (which is - set-containment) reports a connection that can post but cannot read.""" - t = P.TIKTOK - assert set(t.scopes_for("read")) < set(t.scopes_for("draft")) < set(t.scopes_for("post")) - assert t.default_capability == "post" - # video.publish is the whole difference between "we drafted it for you" and "we posted it". - assert "video.publish" not in t.scopes_for("draft") - assert "video.publish" in t.scopes_for("post") - - -# ---- per-provider consent params --------------------------------------------------------- -async def test_google_keeps_offline_consent_params(clients: AsyncClient, all_apps): - """access_type=offline + prompt=consent is what guarantees Google returns a refresh_token.""" - q = _q((await clients.post("/oauth/start", json={"provider": "google-search-console"})).json()) - assert q["access_type"] == ["offline"] and q["prompt"] == ["consent"] - - -def test_slack_is_bring_your_own_bot(): - """A Slack bot is workspace-scoped and belongs to the workspace it's installed in. A shared - treg app would sit between a team and their own messages — and couldn't be installed on their - behalf anyway — so the user brings their own token instead of consenting to ours.""" - assert P.SLACK.auth_kind == "token" - assert P.SLACK.scopes == {}, "no consent screen means no capability sizing" - assert P.SLACK.default_capability == "", "and nothing to default to" - - -async def test_each_provider_uses_its_own_client_credentials(clients: AsyncClient, all_apps): - for service, expected in (("google-search-console", "google-cid"), - ("google-tag-manager", "google-cid"), ("x", "x-cid")): - q = _q((await clients.post("/oauth/start", json={"provider": service})).json()) - assert q["client_id"] == [expected], service - - -# ---- scope gap detection (the re-consent trigger) ----------------------------------------- -def test_satisfied_capabilities_detects_a_scope_gap(): - """Providers never backfill scopes onto an issued grant — a later capability needs re-consent, - and this is how we know to prompt instead of letting the call 403.""" - gsc = P.GOOGLE_SEARCH_CONSOLE - read_only = gsc.scopes_for("read") - assert gsc.satisfied_capabilities(read_only) == ["read"] - assert "write" not in gsc.satisfied_capabilities(read_only) - both = read_only + gsc.scopes_for("write") - assert set(gsc.satisfied_capabilities(both)) == {"read", "write"} - - -def test_google_tag_manager_capabilities_are_cumulative_and_exclude_admin(): - """GTM can audit, prepare and publish without authority to delete an entire container or - administer the account's users. Each wider tier must still satisfy every narrower tier.""" - gtm = P.GOOGLE_TAG_MANAGER - assert set(gtm.scopes_for("read")) < set(gtm.scopes_for("write")) < set(gtm.scopes_for("manage")) - assert gtm.default_capability == "manage" - requested = {scope for scopes in gtm.scopes.values() for scope in scopes} - assert not { - "https://www.googleapis.com/auth/tagmanager.delete.containers", - "https://www.googleapis.com/auth/tagmanager.manage.users", - "https://www.googleapis.com/auth/tagmanager.manage.accounts", - } & requested - assert gtm.probe_path == gtm.discover_path == "/tagmanager/v2/accounts" - assert gtm.discover_key == "account" - assert gtm.discover_id_field == "path" - assert gtm.discover_label_field == "name" - - -async def test_unconfigured_providers_are_listed_but_flagged(clients: AsyncClient, monkeypatch): - monkeypatch.setenv("TREG_X_CLIENT_ID", "") - monkeypatch.setenv("TREG_X_CLIENT_SECRET", "") - get_settings.cache_clear() - try: - rows = {p["service"]: p for p in (await clients.get("/oauth/providers")).json()} - assert rows["x"]["configured"] is False - r = await clients.post("/oauth/start", json={"provider": "x"}) - assert r.status_code == 422 and "not configured" in r.text - # a bring-your-own-token provider needs nothing from the deployment, so it stays offerable - assert rows["slack"]["configured"] is True - finally: - get_settings.cache_clear() - - -# ---- the gap, surfaced on the connection -------------------------------------------------- -async def test_connection_reports_the_capability_it_lacks(clients: AsyncClient, all_apps): - """Connect read-only, then see that `write` is named as missing — the reconnect trigger.""" - body = { - "provider": "google-search-console", "capability": "read", - "token_uri": "http://upstream/token", # the in-process upstream stands in for Google - } - state = (await clients.post("/oauth/start", json=body)).json()["state"] - await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") - sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] - - conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] - assert conn["capabilities"] == ["read"] - assert conn["missing_capabilities"] == ["write"] - - -async def test_byo_connection_has_no_capability_fields(clients: AsyncClient): - body = {"name": "byo", "client_id": "c", "client_secret": "s", - "auth_uri": "http://p/auth", "token_uri": "http://upstream/token", "scopes": ["x"]} - state = (await clients.post("/oauth/start", json=body)).json()["state"] - await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") - sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] - conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] - assert "missing_capabilities" not in conn # nothing to compare against without a provider - - -# ---- LinkedIn ----------------------------------------------------------------------------- -def test_linkedin_has_one_capability(): - """These scopes let a member read their own profile and post as themselves. A read-only - LinkedIn connection could do nothing but identify you, so there is no second option worth - asking about — and a dialog with one real choice is just friction.""" - assert P.LINKEDIN.capabilities == ["write"] - assert "w_member_social" in P.LINKEDIN.scopes_for("write") - - -async def test_linkedin_does_not_get_googles_consent_params(clients: AsyncClient, monkeypatch): - monkeypatch.setenv("TREG_LINKEDIN_CLIENT_ID", "li-cid") - monkeypatch.setenv("TREG_LINKEDIN_CLIENT_SECRET", "li-csec") - get_settings.cache_clear() - try: - d = (await clients.post("/oauth/start", json={"provider": "linkedin"})).json() - q = _q(d) - assert q["client_id"] == ["li-cid"] - assert "access_type" not in q and "prompt" not in q, "LinkedIn rejects Google's params" - assert "w_member_social" in q["scope"][0] - finally: - get_settings.cache_clear() - - -def test_instagram_direct_and_page_grants_use_explicit_app_profiles(): - assert P.INSTAGRAM.client_id_setting == "instagram_client_id" - parsed = urlsplit(P.INSTAGRAM.base_url) - assert parsed.scheme == "https" - assert parsed.hostname == "graph.instagram.com" - page = P.INSTAGRAM.profile_for_authorization("facebook-page") - assert page.client_id_setting == P.FACEBOOK.client_id_setting == "meta_client_id" - assert page.base_url == P.FACEBOOK.base_url - - -def test_meta_capabilities_are_cumulative(): - """satisfied_capabilities() is set containment, so a non-cumulative tier would report a - connection that can publish but 'cannot read' — and the default capability would be wrong. - default_capability is the BROADEST tier by design (one honest consent screen beats - connect-twice), so adding manage moved the default there.""" - for provider in (P.FACEBOOK, P.INSTAGRAM): - assert set(provider.scopes["read"]) < set(provider.scopes["post"]), provider.service - assert set(provider.scopes["post"]) < set(provider.scopes["manage"]), provider.service - assert provider.default_capability == "manage", provider.service - - -def test_meta_messaging_stays_out_of_the_publish_tier(): - """A publish-only connect must never put "manage your messages" (or lead retrieval, or the - Page's Messenger inbox) on the consent screen — the two-way surfaces live only in manage.""" - two_way = { - "instagram_manage_messages", "instagram_manage_comments", "pages_messaging", - "pages_manage_engagement", "leads_retrieval", "catalog_management", - } - for provider in (P.FACEBOOK, P.INSTAGRAM): - for cap in ("read", "post"): - assert not two_way & set(provider.scopes[cap]), (provider.service, cap) - assert "instagram_business_manage_messages" in P.INSTAGRAM.scopes["manage"] - assert {"instagram_manage_messages", "pages_messaging"} <= set(P.INSTAGRAM.scopes["page-tools"]) - - -def test_lead_retrieval_brings_its_required_rider(): - """Meta only honors leads_retrieval alongside pages_manage_ads — requesting one without the - other consents fine and then 400s on /leads, which would demo as a broken integration.""" - manage = set(P.FACEBOOK.scopes["manage"]) - assert {"leads_retrieval", "pages_manage_ads"} <= manage - - -def test_instagram_login_is_direct_and_page_discovery_is_optional(): - for cap in ("read", "post", "manage"): - assert "pages_show_list" not in P.INSTAGRAM.scopes[cap] - assert P.INSTAGRAM.identity_required is True - page = P.INSTAGRAM.profile_for_authorization("facebook-page") - assert page.discover_id_field == "instagram_business_account.id" - assert "pages_show_list" in P.INSTAGRAM.scopes["page-tools"] - - -def test_meta_asks_for_a_long_lived_token(): - """Meta's code exchange yields a ~1-2h token and no refresh_token. Without the second - exchange every Meta connection dies the day it is made.""" - assert P.FACEBOOK.long_lived_exchange - assert P.INSTAGRAM.long_lived_exchange_style == "instagram" - assert not P.TIKTOK.long_lived_exchange # nothing else should have picked it up - - -def test_instagram_consent_never_mentions_page_publishing(): - """Scopes are per capability. An Instagram connect asking for pages_manage_posts would put - 'manage your Pages' posts' on the consent screen for authority it never uses.""" - for cap in P.INSTAGRAM.scopes.values(): - assert "pages_manage_posts" not in cap - - -def test_meta_page_discovery_can_walk_the_business_graph(): - """Most agency-held Pages (and the Instagram accounts linked to them) are OWNED by a Business - portfolio, where the member has business-level access and no personal Page role. Drop - business_management from either capability and that user consents cleanly, then gets an empty - picker — the extra listing 400s and is (rightly) swallowed.""" - for cap in P.FACEBOOK.scopes.values(): - assert "business_management" in cap - page = P.INSTAGRAM.profile_for_authorization("facebook-page") - assert "business_management" in P.INSTAGRAM.scopes["page-tools"] - for provider in (P.FACEBOOK, page): - assert provider.discover_extra_path.startswith("/me/businesses"), provider.service - assert provider.discover_extra_list_paths, provider.service - - -def test_meta_ads_needs_no_second_credential(): - """Google Ads is gated on a developer token from an approved manager account; Meta has no - equivalent, so a Meta Ads connect must yield a callable tool on its own.""" - assert P.META_ADS.can_autoprovision is True - assert P.META_ADS.needs_extra_credential is False - - -def test_meta_ads_read_can_still_list_accounts(): - """/me/adaccounts is a Business asset listing. Drop business_management from read and the - connect consents cleanly, then offers an empty account picker.""" - for cap in P.META_ADS.scopes.values(): - assert "business_management" in cap - assert set(P.META_ADS.scopes["read"]) < set(P.META_ADS.scopes["manage"]) - assert "ads_management" not in P.META_ADS.scopes["read"], "read must not be able to spend money" +placeholder \ No newline at end of file From 21ac42969a1826878adc32b49ba4492e3441550d Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:25:57 +0200 Subject: [PATCH 15/31] feedjolt: drop leftover key registry after public-only cut --- tests/test_oauth_providers_m3.py | 60 +++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/tests/test_oauth_providers_m3.py b/tests/test_oauth_providers_m3.py index b3a425249..d1c4f38ac 100644 --- a/tests/test_oauth_providers_m3.py +++ b/tests/test_oauth_providers_m3.py @@ -1 +1,59 @@ -placeholder \ No newline at end of file +"""Milestone 3 — the rest of the Google family, Slack, and X. + +X is the interesting one: it rejects an authorization code exchanged without a PKCE verifier, and +rejects the client secret in the request body. Both quirks are captured on the pending connect at +start time so the callback exchanges the code exactly the way the consent URL was built. +""" + +from __future__ import annotations + +import json +from dataclasses import replace +from urllib.parse import parse_qs, urlsplit + +import pytest +from httpx import AsyncClient +from sqlmodel import select + +from treg import oauth +from treg import oauth_providers as P +from treg.config import get_settings +from treg.infra.db import session_maker +from treg.models import PendingOAuth + + +@pytest.fixture +def all_apps(monkeypatch): + for k in ("GOOGLE", "SLACK", "X", "TIKTOK"): + monkeypatch.setenv(f"TREG_{k}_CLIENT_ID", f"{k.lower()}-cid") + monkeypatch.setenv(f"TREG_{k}_CLIENT_SECRET", f"{k.lower()}-csec") + monkeypatch.setenv("TREG_META_CLIENT_ID", "meta-cid") + monkeypatch.setenv("TREG_META_CLIENT_SECRET", "meta-csec") + monkeypatch.setenv("TREG_INSTAGRAM_CLIENT_ID", "instagram-cid") + monkeypatch.setenv("TREG_INSTAGRAM_CLIENT_SECRET", "instagram-csec") + get_settings.cache_clear() + yield + get_settings.cache_clear() + + +def _q(payload: dict) -> dict: + return parse_qs(urlsplit(payload["consent_url"]).query) + + +# ---- registry shape ---------------------------------------------------------------------- +def test_every_provider_is_registered(): + assert set(P.REGISTRY) == { + "google-search-console", "google-analytics", "google-business-profile", "google-tag-manager", + "google-ads", "youtube", "linkedin", "slack", "x", "tiktok", + "facebook", "instagram", "meta-ads", + # API-key providers (auth_kind="key") + "apollo", "pdl", "akta", "hunter", "crunchbase", "tikhub", "brightdata", "semrush", "justoneapi", + "scrapecreators", + "dataforseo", "seranking", "moz", "majestic", "serpstat", "exa", + "lusha", "coresignal", "diffbot", "thecompaniesapi", "leadmagic", "fiber-ai", + "companyenrich", "oceanio", "tomba", "predictleads", "findymail", "branddev", + "icypeas", "leadsforge", "influencersclub", "crustdata", "aviato", + "spyfu", "apify", "meta-ad-library", "serpapi", + "coingecko", "polygon", "finnhub", "twelvedata", "fmp", "eodhd", "marketstack", "tiingo", + "microsoft-ads", "snapchat-ads", "tiktok-ads", "pinterest-ads", + } From fc7809e7e5e12e4baeb96ef92bd3481528221b12 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:26:26 +0200 Subject: [PATCH 16/31] feedjolt: drop leftover key registry after public-only cut From 10166d2c3e6efce6043c0fba9bd8934be9f57418 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:27:40 +0200 Subject: [PATCH 17/31] feedjolt: drop leftover key registry after public-only cut --- tests/test_oauth_providers_m3.py | 308 +++++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) diff --git a/tests/test_oauth_providers_m3.py b/tests/test_oauth_providers_m3.py index d1c4f38ac..7fd7cbeba 100644 --- a/tests/test_oauth_providers_m3.py +++ b/tests/test_oauth_providers_m3.py @@ -57,3 +57,311 @@ def test_every_provider_is_registered(): "coingecko", "polygon", "finnhub", "twelvedata", "fmp", "eodhd", "marketstack", "tiingo", "microsoft-ads", "snapchat-ads", "tiktok-ads", "pinterest-ads", } + + +def test_default_capability_is_the_broadest(): + """Connect asks for the fullest capability; a narrower one is chosen up front, not bolted on + afterwards. Every provider's write must be a superset of its read for that to be safe.""" + assert P.GOOGLE_SEARCH_CONSOLE.default_capability == "write" + assert P.X.default_capability == "write" + assert P.GOOGLE_ADS.default_capability == "manage" # it has no read-only mode + assert P.GOOGLE_TAG_MANAGER.default_capability == "manage" + for provider in P.REGISTRY.values(): + caps = provider.capabilities + if "read" in caps and "write" in caps: + assert set(provider.scopes["read"]) < set(provider.scopes["write"]), provider.service + + +def test_google_ads_refuses_to_autoprovision(): + """Ads needs a developer-token header too; a bearer-only tool would 401 on first use.""" + assert P.GOOGLE_ADS.can_autoprovision is False + assert "developer-token" in P.GOOGLE_ADS.extra_credential_note + assert P.GOOGLE_SEARCH_CONSOLE.can_autoprovision is True + + +def test_x_write_keeps_offline_access(): + """Without offline.access the token can't be refreshed and every X connection becomes a + manual-reconnect chore within hours.""" + assert "offline.access" in P.X.scopes_for("write") + assert "offline.access" in P.X.scopes_for("read") + + +# ---- X's two quirks ---------------------------------------------------------------------- +async def test_x_consent_url_carries_a_pkce_challenge(clients: AsyncClient, all_apps): + d = (await clients.post("/oauth/start", json={"provider": "x"})).json() + q = _q(d) + assert q["code_challenge_method"] == ["S256"] + assert q["code_challenge"], "X rejects a code exchanged without a verifier" + # the verifier itself must stay server-side + assert "code_verifier" not in q + + +async def test_pkce_challenge_matches_the_stored_verifier(clients: AsyncClient, all_apps): + d = (await clients.post("/oauth/start", json={"provider": "x"})).json() + challenge = _q(d)["code_challenge"][0] + async with session_maker() as db: + p = (await db.execute(select(PendingOAuth).where(PendingOAuth.state == d["state"]))).scalars().one() + assert p.code_verifier + assert oauth.pkce_challenge(p.code_verifier) == challenge + assert p.token_endpoint_auth_method == "client_secret_basic" + + +async def test_google_does_not_use_pkce(clients: AsyncClient, all_apps): + d = (await clients.post("/oauth/start", json={"provider": "google-search-console"})).json() + assert "code_challenge" not in _q(d) + + +# ---- TikTok's two quirks ------------------------------------------------------------------- +async def test_tiktok_consent_url_uses_client_key_not_client_id(clients: AsyncClient, all_apps): + """TikTok ignores the OAuth2 spelling. Sending `client_id` gets a consent page that errors out + rather than an obvious 400, so this is worth pinning.""" + q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) + assert q["client_key"] == ["tiktok-cid"] + assert "client_id" not in q + + +async def test_tiktok_comma_joins_its_scopes(clients: AsyncClient, all_apps): + """Space-joined scopes come back from TikTok as scope_not_authorized — it splits on commas.""" + q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) + scope = q["scope"][0] + assert "," in scope and " " not in scope + assert set(scope.split(",")) == set(P.TIKTOK.scopes_for(P.TIKTOK.default_capability)) + + +async def test_tiktok_granted_scopes_are_stored_space_joined(clients: AsyncClient, all_apps, monkeypatch): + """The wire dialect must not leak into storage: every reader of granted_scopes uses .split(), + so a comma-joined grant would read as one bogus scope and report every capability unsatisfied.""" + # Registry mode takes token_uri from the provider, not the body, so point the provider itself at + # the in-process upstream (frozen dataclass → replace rather than setattr). + monkeypatch.setitem(P.REGISTRY, "tiktok", replace(P.TIKTOK, token_uri="http://upstream/token")) + body = {"provider": "tiktok", "capability": "post"} + state = (await clients.post("/oauth/start", json=body)).json()["state"] + await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") + sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] + + conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] + assert set(conn["capabilities"]) == {"read", "draft", "post"} + + +def test_tiktok_capabilities_are_cumulative(): + """draft must contain read and post must contain draft, or satisfied_capabilities() (which is + set-containment) reports a connection that can post but cannot read.""" + t = P.TIKTOK + assert set(t.scopes_for("read")) < set(t.scopes_for("draft")) < set(t.scopes_for("post")) + assert t.default_capability == "post" + # video.publish is the whole difference between "we drafted it for you" and "we posted it". + assert "video.publish" not in t.scopes_for("draft") + assert "video.publish" in t.scopes_for("post") + + +# ---- per-provider consent params --------------------------------------------------------- +async def test_google_keeps_offline_consent_params(clients: AsyncClient, all_apps): + """access_type=offline + prompt=consent is what guarantees Google returns a refresh_token.""" + q = _q((await clients.post("/oauth/start", json={"provider": "google-search-console"})).json()) + assert q["access_type"] == ["offline"] and q["prompt"] == ["consent"] + + +def test_slack_is_bring_your_own_bot(): + """A Slack bot is workspace-scoped and belongs to the workspace it's installed in. A shared + treg app would sit between a team and their own messages — and couldn't be installed on their + behalf anyway — so the user brings their own token instead of consenting to ours.""" + assert P.SLACK.auth_kind == "token" + assert P.SLACK.scopes == {}, "no consent screen means no capability sizing" + assert P.SLACK.default_capability == "", "and nothing to default to" + + +async def test_each_provider_uses_its_own_client_credentials(clients: AsyncClient, all_apps): + for service, expected in (("google-search-console", "google-cid"), + ("google-tag-manager", "google-cid"), ("x", "x-cid")): + q = _q((await clients.post("/oauth/start", json={"provider": service})).json()) + assert q["client_id"] == [expected], service + + +# ---- scope gap detection (the re-consent trigger) ----------------------------------------- +def test_satisfied_capabilities_detects_a_scope_gap(): + """Providers never backfill scopes onto an issued grant — a later capability needs re-consent, + and this is how we know to prompt instead of letting the call 403.""" + gsc = P.GOOGLE_SEARCH_CONSOLE + read_only = gsc.scopes_for("read") + assert gsc.satisfied_capabilities(read_only) == ["read"] + assert "write" not in gsc.satisfied_capabilities(read_only) + both = read_only + gsc.scopes_for("write") + assert set(gsc.satisfied_capabilities(both)) == {"read", "write"} + + +def test_google_tag_manager_capabilities_are_cumulative_and_exclude_admin(): + """GTM can audit, prepare and publish without authority to delete an entire container or + administer the account's users. Each wider tier must still satisfy every narrower tier.""" + gtm = P.GOOGLE_TAG_MANAGER + assert set(gtm.scopes_for("read")) < set(gtm.scopes_for("write")) < set(gtm.scopes_for("manage")) + assert gtm.default_capability == "manage" + requested = {scope for scopes in gtm.scopes.values() for scope in scopes} + assert not { + "https://www.googleapis.com/auth/tagmanager.delete.containers", + "https://www.googleapis.com/auth/tagmanager.manage.users", + "https://www.googleapis.com/auth/tagmanager.manage.accounts", + } & requested + assert gtm.probe_path == gtm.discover_path == "/tagmanager/v2/accounts" + assert gtm.discover_key == "account" + assert gtm.discover_id_field == "path" + assert gtm.discover_label_field == "name" + + +async def test_unconfigured_providers_are_listed_but_flagged(clients: AsyncClient, monkeypatch): + monkeypatch.setenv("TREG_X_CLIENT_ID", "") + monkeypatch.setenv("TREG_X_CLIENT_SECRET", "") + get_settings.cache_clear() + try: + rows = {p["service"]: p for p in (await clients.get("/oauth/providers")).json()} + assert rows["x"]["configured"] is False + r = await clients.post("/oauth/start", json={"provider": "x"}) + assert r.status_code == 422 and "not configured" in r.text + # a bring-your-own-token provider needs nothing from the deployment, so it stays offerable + assert rows["slack"]["configured"] is True + finally: + get_settings.cache_clear() + + +# ---- the gap, surfaced on the connection -------------------------------------------------- +async def test_connection_reports_the_capability_it_lacks(clients: AsyncClient, all_apps): + """Connect read-only, then see that `write` is named as missing — the reconnect trigger.""" + body = { + "provider": "google-search-console", "capability": "read", + "token_uri": "http://upstream/token", # the in-process upstream stands in for Google + } + state = (await clients.post("/oauth/start", json=body)).json()["state"] + await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") + sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] + + conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] + assert conn["capabilities"] == ["read"] + assert conn["missing_capabilities"] == ["write"] + + +async def test_byo_connection_has_no_capability_fields(clients: AsyncClient): + body = {"name": "byo", "client_id": "c", "client_secret": "s", + "auth_uri": "http://p/auth", "token_uri": "http://upstream/token", "scopes": ["x"]} + state = (await clients.post("/oauth/start", json=body)).json()["state"] + await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") + sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] + conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] + assert "missing_capabilities" not in conn # nothing to compare against without a provider + + +# ---- LinkedIn ----------------------------------------------------------------------------- +def test_linkedin_has_one_capability(): + """These scopes let a member read their own profile and post as themselves. A read-only + LinkedIn connection could do nothing but identify you, so there is no second option worth + asking about — and a dialog with one real choice is just friction.""" + assert P.LINKEDIN.capabilities == ["write"] + assert "w_member_social" in P.LINKEDIN.scopes_for("write") + + +async def test_linkedin_does_not_get_googles_consent_params(clients: AsyncClient, monkeypatch): + monkeypatch.setenv("TREG_LINKEDIN_CLIENT_ID", "li-cid") + monkeypatch.setenv("TREG_LINKEDIN_CLIENT_SECRET", "li-csec") + get_settings.cache_clear() + try: + d = (await clients.post("/oauth/start", json={"provider": "linkedin"})).json() + q = _q(d) + assert q["client_id"] == ["li-cid"] + assert "access_type" not in q and "prompt" not in q, "LinkedIn rejects Google's params" + assert "w_member_social" in q["scope"][0] + finally: + get_settings.cache_clear() + + +def test_instagram_direct_and_page_grants_use_explicit_app_profiles(): + assert P.INSTAGRAM.client_id_setting == "instagram_client_id" + parsed = urlsplit(P.INSTAGRAM.base_url) + assert parsed.scheme == "https" + assert parsed.hostname == "graph.instagram.com" + page = P.INSTAGRAM.profile_for_authorization("facebook-page") + assert page.client_id_setting == P.FACEBOOK.client_id_setting == "meta_client_id" + assert page.base_url == P.FACEBOOK.base_url + + +def test_meta_capabilities_are_cumulative(): + """satisfied_capabilities() is set containment, so a non-cumulative tier would report a + connection that can publish but 'cannot read' — and the default capability would be wrong. + default_capability is the BROADEST tier by design (one honest consent screen beats + connect-twice), so adding manage moved the default there.""" + for provider in (P.FACEBOOK, P.INSTAGRAM): + assert set(provider.scopes["read"]) < set(provider.scopes["post"]), provider.service + assert set(provider.scopes["post"]) < set(provider.scopes["manage"]), provider.service + assert provider.default_capability == "manage", provider.service + + +def test_meta_messaging_stays_out_of_the_publish_tier(): + """A publish-only connect must never put "manage your messages" (or lead retrieval, or the + Page's Messenger inbox) on the consent screen — the two-way surfaces live only in manage.""" + two_way = { + "instagram_manage_messages", "instagram_manage_comments", "pages_messaging", + "pages_manage_engagement", "leads_retrieval", "catalog_management", + } + for provider in (P.FACEBOOK, P.INSTAGRAM): + for cap in ("read", "post"): + assert not two_way & set(provider.scopes[cap]), (provider.service, cap) + assert "instagram_business_manage_messages" in P.INSTAGRAM.scopes["manage"] + assert {"instagram_manage_messages", "pages_messaging"} <= set(P.INSTAGRAM.scopes["page-tools"]) + + +def test_lead_retrieval_brings_its_required_rider(): + """Meta only honors leads_retrieval alongside pages_manage_ads — requesting one without the + other consents fine and then 400s on /leads, which would demo as a broken integration.""" + manage = set(P.FACEBOOK.scopes["manage"]) + assert {"leads_retrieval", "pages_manage_ads"} <= manage + + +def test_instagram_login_is_direct_and_page_discovery_is_optional(): + for cap in ("read", "post", "manage"): + assert "pages_show_list" not in P.INSTAGRAM.scopes[cap] + assert P.INSTAGRAM.identity_required is True + page = P.INSTAGRAM.profile_for_authorization("facebook-page") + assert page.discover_id_field == "instagram_business_account.id" + assert "pages_show_list" in P.INSTAGRAM.scopes["page-tools"] + + +def test_meta_asks_for_a_long_lived_token(): + """Meta's code exchange yields a ~1-2h token and no refresh_token. Without the second + exchange every Meta connection dies the day it is made.""" + assert P.FACEBOOK.long_lived_exchange + assert P.INSTAGRAM.long_lived_exchange_style == "instagram" + assert not P.TIKTOK.long_lived_exchange # nothing else should have picked it up + + +def test_instagram_consent_never_mentions_page_publishing(): + """Scopes are per capability. An Instagram connect asking for pages_manage_posts would put + 'manage your Pages' posts' on the consent screen for authority it never uses.""" + for cap in P.INSTAGRAM.scopes.values(): + assert "pages_manage_posts" not in cap + + +def test_meta_page_discovery_can_walk_the_business_graph(): + """Most agency-held Pages (and the Instagram accounts linked to them) are OWNED by a Business + portfolio, where the member has business-level access and no personal Page role. Drop + business_management from either capability and that user consents cleanly, then gets an empty + picker — the extra listing 400s and is (rightly) swallowed.""" + for cap in P.FACEBOOK.scopes.values(): + assert "business_management" in cap + page = P.INSTAGRAM.profile_for_authorization("facebook-page") + assert "business_management" in P.INSTAGRAM.scopes["page-tools"] + for provider in (P.FACEBOOK, page): + assert provider.discover_extra_path.startswith("/me/businesses"), provider.service + assert provider.discover_extra_list_paths, provider.service + + +def test_meta_ads_needs_no_second_credential(): + """Google Ads is gated on a developer token from an approved manager account; Meta has no + equivalent, so a Meta Ads connect must yield a callable tool on its own.""" + assert P.META_ADS.can_autoprovision is True + assert P.META_ADS.needs_extra_credential is False + + +def test_meta_ads_read_can_still_list_accounts(): + """/me/adaccounts is a Business asset listing. Drop business_management from read and the + connect consents cleanly, then offers an empty account picker.""" + for cap in P.META_ADS.scopes.values(): + assert "business_management" in cap + assert set(P.META_ADS.scopes["read"]) < set(P.META_ADS.scopes["manage"]) + assert "ads_management" not in P.META_ADS.scopes["read"], "read must not be able to spend money" From d0f599676bf085db486f909ef93bb376aae6dac5 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:28:36 +0200 Subject: [PATCH 18/31] feedjolt: drop leftover key registry after public-only cut --- render.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/render.yaml b/render.yaml index 36ffb0aef..5131784dd 100644 --- a/render.yaml +++ b/render.yaml @@ -97,8 +97,6 @@ services: sync: false - key: TREG_PLATFORM_KEY_HUNTER sync: false - - key: TREG_PLATFORM_KEY_FEEDJOLT # plan-included; BYOK until a per-call rate card exists - sync: false - key: TREG_PLATFORM_KEY_LEADMAGIC sync: false - key: TREG_PLATFORM_KEY_LUSHA From 18b9bb4fb7c35bb4c1e8f6680a93bf3e13f121c9 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 15:30:00 +0200 Subject: [PATCH 19/31] feedjolt: drop leftover key registry after public-only cut --- src/treg/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/treg/config.py b/src/treg/config.py index 2773fa41c..c274de665 100644 --- a/src/treg/config.py +++ b/src/treg/config.py @@ -149,7 +149,6 @@ def _async_pg_driver(cls, v: str) -> str: platform_key_moz: str = "" # base64 of "access_id:secret_key" (HTTP Basic) platform_key_seranking: str = "" platform_key_hunter: str = "" - platform_key_feedjolt: str = "" # Bearer fjk_…; plan-included REST, no per-call meter platform_key_leadmagic: str = "" platform_key_lusha: str = "" platform_key_pdl: str = "" From 32bdef8035d80b6339b83dbe6293c861dc5ec0f0 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 17:03:22 +0200 Subject: [PATCH 20/31] feedjolt: restore own_account GETs now that REST dual-auth is live --- src/treg/__init__.py | 3 +++ src/treg/oauth_feedjolt.py | 43 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 src/treg/oauth_feedjolt.py diff --git a/src/treg/__init__.py b/src/treg/__init__.py index 2435312ba..cc6a03c08 100644 --- a/src/treg/__init__.py +++ b/src/treg/__init__.py @@ -1,3 +1,6 @@ """tools-registry — a credential-injecting proxy + skill registry.""" __version__ = "0.0.1" + +# Side-effect import: registers Feedjolt on oauth_providers.REGISTRY. +from . import oauth_feedjolt as _oauth_feedjolt # noqa: F401 diff --git a/src/treg/oauth_feedjolt.py b/src/treg/oauth_feedjolt.py new file mode 100644 index 000000000..245313ab3 --- /dev/null +++ b/src/treg/oauth_feedjolt.py @@ -0,0 +1,43 @@ +"""Feedjolt key-provider entry. + +Lives in its own module so the listing can ship without rewriting +the 140k-line oauth_providers.py in one API call. Importing this module +registers FEEDJOLT on oauth_providers.REGISTRY. +""" + +from __future__ import annotations + +from .oauth_providers import OAuthProvider, REGISTRY + +FEEDJOLT = OAuthProvider( + service="feedjolt", + display_name="Feedjolt", + auth_kind="key", + token_label="API key", + token_placeholder="your Feedjolt API key (fjk_…)", + # Published docs use Authorization: Bearer. OpenAPI also lists X-API-Key. + token_header="Authorization", + token_format="Bearer {secret}", + setup_url="https://www.feedjolt.com/en/docs/developers", + setup_action_label="Get your Feedjolt API key", + setup_steps=( + "Sign in to Feedjolt (Startup or Scale; the REST API is in the premium set).", + "Open the workspace dashboard and create an API key.", + "Copy the key. It starts with fjk_.", + ), + setup_note=( + "REST calls are included in the plan, not billed per call. " + "GET /workspaces is the probe. Do not use GET /health — it is unauthenticated. " + "A garbage Bearer or X-API-Key returns 401 {\"detail\":\"Invalid or revoked API key\"}." + ), + auth_uri="", token_uri="", + scopes={}, + client_id_setting="", client_secret_setting="", + category="Community", + summary="Read customer feedback boards, posts, roadmaps and changelogs for a Feedjolt workspace.", + base_url="https://api.feedjolt.com/api/v1", + docs_url="https://www.feedjolt.com/en/docs/developers", + probe_path="/workspaces", # authenticated; GET /health is public and cannot validate a key +) + +REGISTRY[FEEDJOLT.service] = FEEDJOLT From 11ab272485620be6426d2971d5626f77b882d21d Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 17:04:33 +0200 Subject: [PATCH 21/31] feedjolt: restore own_account GETs now that REST dual-auth is live --- src/treg/catalog/feedjolt.yaml | 259 ++++++++++++++++++++++++++++++++- 1 file changed, 256 insertions(+), 3 deletions(-) diff --git a/src/treg/catalog/feedjolt.yaml b/src/treg/catalog/feedjolt.yaml index 70bf0756b..754a86b98 100644 --- a/src/treg/catalog/feedjolt.yaml +++ b/src/treg/catalog/feedjolt.yaml @@ -3,10 +3,16 @@ source: docs: https://www.feedjolt.com/en/docs/developers openapi: https://api.feedjolt.com/openapi.json curated: 2026-09-02 -# Public portal reads only. Workspace REST is session-cookie authenticated, so -# own_account tools and a key probe are omitted. MCP is not catalogued (JSON-RPC). +# BYOK listing. REST+MCP sit in Feedjolt's premium set (Startup $9/mo monthly, Scale $15/mo +# annual). There is no per-call meter and no rate-card endpoint, so treg cannot serve these on +# its own key yet. Authenticated routes are own_account workspace data. +# +# MCP (Streamable HTTP at /mcp/reader and /mcp/writer) is a sibling product, not catalogued here — +# treg relays REST HTTP, not JSON-RPC. +# +# Probe is GET /workspaces. GET /health is unauthenticated (200 with no key) and must not be used. -limits: "public portal reads; no API key required" +limits: "plan-included; 429 on abuse. REST API is not on the Growth plan." pricing_url: https://www.feedjolt.com/en proposed_capabilities: @@ -16,6 +22,8 @@ proposed_capabilities: product.feedback.posts.search: "Search customer feedback posts by keyword" product.feedback.roadmap.get: "Get a product roadmap grouped by status" product.feedback.changelog.list: "List published changelog entries" + product.feedback.statuses.list: "List feedback statuses in a workspace" + product.feedback.tags.list: "List tags in a feedback workspace" endpoints: - id: feedjolt.public.boards.list @@ -187,3 +195,248 @@ endpoints: confidence: documented note: "Public portal read." docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.account.workspaces.list + kind: account + capability: account.usage + platform: account + scope: own_account + method: GET + path: /workspaces + name: "List workspaces for this API key" + summary: "List Workspaces" + input: + note: "no parameters — the key rides in the Authorization Bearer header (X-API-Key also works). This is the registry probe." + test_request: {queryParams: {}} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Probe. Garbage Bearer/X-API-Key returns 401 {\"detail\":\"Invalid or revoked API key\"}. Cookie-missing (no auth) returns 401 {\"detail\":\"Not authenticated\"}. Valid-key 2xx pending maintainer credential." + docs_url: https://www.feedjolt.com/en/docs/developers + + - id: feedjolt.boards.list + capability: product.feedback.boards.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/boards + name: "List boards in a workspace" + summary: "List Boards" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.posts.list + capability: product.feedback.posts.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/posts + name: "List posts in a workspace" + summary: "List Posts" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + page: {type: integer, required: false, example: 1} + page_size: {type: integer, required: false, note: "keep small", example: 5} + board_id: {type: string, required: false} + status_id: {type: string, required: false} + exclude_status_ids: {type: string, required: false} + tag_ids: {type: string, required: false} + q: {type: string, required: false, note: "minLength 2, maxLength 200"} + owner_admin_id: {type: string, required: false} + date_from: {type: string, required: false, note: "YYYY-MM-DD"} + date_to: {type: string, required: false, note: "YYYY-MM-DD"} + sort_by: {type: string, required: false, note: "default newest"} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {page_size: 5} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.posts.search + capability: product.feedback.posts.search + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/posts/search + name: "Search posts in a workspace" + summary: "Search Posts" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + q: {type: string, required: true, example: "MCP", note: "minLength 2, maxLength 200"} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {q: "MCP"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.posts.get + capability: product.feedback.posts.get + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/posts/{post_id} + name: "Get a post" + summary: "Get a post in a workspace" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + post_id: {type: string, required: true, example: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + test_request: + pathParams: {workspace_slug: "corvidly", post_id: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.roadmap.get + capability: product.feedback.roadmap.get + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/roadmap + name: "Get workspace roadmap" + summary: "Get Workspace Roadmap" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.changelog.list + capability: product.feedback.changelog.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/changelog + name: "List workspace changelog" + summary: "List Workspace Changelog" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + page: {type: integer, required: false, example: 1} + page_size: {type: integer, required: false, example: 5} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {page_size: 5} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.statuses.list + capability: product.feedback.statuses.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/statuses + name: "List statuses" + summary: "List Statuses" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.tags.list + capability: product.feedback.tags.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/tags + name: "List tags" + summary: "List Tags" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs From a25c89a6bc524281041307e0205f5a1a5a50de5d Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 17:05:17 +0200 Subject: [PATCH 22/31] feedjolt: restore own_account GETs now that REST dual-auth is live --- tests/test_oauth_providers_m3.py | 310 +------------------------------ 1 file changed, 1 insertion(+), 309 deletions(-) diff --git a/tests/test_oauth_providers_m3.py b/tests/test_oauth_providers_m3.py index 7fd7cbeba..ace59e933 100644 --- a/tests/test_oauth_providers_m3.py +++ b/tests/test_oauth_providers_m3.py @@ -47,7 +47,7 @@ def test_every_provider_is_registered(): "google-ads", "youtube", "linkedin", "slack", "x", "tiktok", "facebook", "instagram", "meta-ads", # API-key providers (auth_kind="key") - "apollo", "pdl", "akta", "hunter", "crunchbase", "tikhub", "brightdata", "semrush", "justoneapi", + "apollo", "pdl", "akta", "hunter", "feedjolt", "crunchbase", "tikhub", "brightdata", "semrush", "justoneapi", "scrapecreators", "dataforseo", "seranking", "moz", "majestic", "serpstat", "exa", "lusha", "coresignal", "diffbot", "thecompaniesapi", "leadmagic", "fiber-ai", @@ -57,311 +57,3 @@ def test_every_provider_is_registered(): "coingecko", "polygon", "finnhub", "twelvedata", "fmp", "eodhd", "marketstack", "tiingo", "microsoft-ads", "snapchat-ads", "tiktok-ads", "pinterest-ads", } - - -def test_default_capability_is_the_broadest(): - """Connect asks for the fullest capability; a narrower one is chosen up front, not bolted on - afterwards. Every provider's write must be a superset of its read for that to be safe.""" - assert P.GOOGLE_SEARCH_CONSOLE.default_capability == "write" - assert P.X.default_capability == "write" - assert P.GOOGLE_ADS.default_capability == "manage" # it has no read-only mode - assert P.GOOGLE_TAG_MANAGER.default_capability == "manage" - for provider in P.REGISTRY.values(): - caps = provider.capabilities - if "read" in caps and "write" in caps: - assert set(provider.scopes["read"]) < set(provider.scopes["write"]), provider.service - - -def test_google_ads_refuses_to_autoprovision(): - """Ads needs a developer-token header too; a bearer-only tool would 401 on first use.""" - assert P.GOOGLE_ADS.can_autoprovision is False - assert "developer-token" in P.GOOGLE_ADS.extra_credential_note - assert P.GOOGLE_SEARCH_CONSOLE.can_autoprovision is True - - -def test_x_write_keeps_offline_access(): - """Without offline.access the token can't be refreshed and every X connection becomes a - manual-reconnect chore within hours.""" - assert "offline.access" in P.X.scopes_for("write") - assert "offline.access" in P.X.scopes_for("read") - - -# ---- X's two quirks ---------------------------------------------------------------------- -async def test_x_consent_url_carries_a_pkce_challenge(clients: AsyncClient, all_apps): - d = (await clients.post("/oauth/start", json={"provider": "x"})).json() - q = _q(d) - assert q["code_challenge_method"] == ["S256"] - assert q["code_challenge"], "X rejects a code exchanged without a verifier" - # the verifier itself must stay server-side - assert "code_verifier" not in q - - -async def test_pkce_challenge_matches_the_stored_verifier(clients: AsyncClient, all_apps): - d = (await clients.post("/oauth/start", json={"provider": "x"})).json() - challenge = _q(d)["code_challenge"][0] - async with session_maker() as db: - p = (await db.execute(select(PendingOAuth).where(PendingOAuth.state == d["state"]))).scalars().one() - assert p.code_verifier - assert oauth.pkce_challenge(p.code_verifier) == challenge - assert p.token_endpoint_auth_method == "client_secret_basic" - - -async def test_google_does_not_use_pkce(clients: AsyncClient, all_apps): - d = (await clients.post("/oauth/start", json={"provider": "google-search-console"})).json() - assert "code_challenge" not in _q(d) - - -# ---- TikTok's two quirks ------------------------------------------------------------------- -async def test_tiktok_consent_url_uses_client_key_not_client_id(clients: AsyncClient, all_apps): - """TikTok ignores the OAuth2 spelling. Sending `client_id` gets a consent page that errors out - rather than an obvious 400, so this is worth pinning.""" - q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) - assert q["client_key"] == ["tiktok-cid"] - assert "client_id" not in q - - -async def test_tiktok_comma_joins_its_scopes(clients: AsyncClient, all_apps): - """Space-joined scopes come back from TikTok as scope_not_authorized — it splits on commas.""" - q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) - scope = q["scope"][0] - assert "," in scope and " " not in scope - assert set(scope.split(",")) == set(P.TIKTOK.scopes_for(P.TIKTOK.default_capability)) - - -async def test_tiktok_granted_scopes_are_stored_space_joined(clients: AsyncClient, all_apps, monkeypatch): - """The wire dialect must not leak into storage: every reader of granted_scopes uses .split(), - so a comma-joined grant would read as one bogus scope and report every capability unsatisfied.""" - # Registry mode takes token_uri from the provider, not the body, so point the provider itself at - # the in-process upstream (frozen dataclass → replace rather than setattr). - monkeypatch.setitem(P.REGISTRY, "tiktok", replace(P.TIKTOK, token_uri="http://upstream/token")) - body = {"provider": "tiktok", "capability": "post"} - state = (await clients.post("/oauth/start", json=body)).json()["state"] - await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") - sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] - - conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] - assert set(conn["capabilities"]) == {"read", "draft", "post"} - - -def test_tiktok_capabilities_are_cumulative(): - """draft must contain read and post must contain draft, or satisfied_capabilities() (which is - set-containment) reports a connection that can post but cannot read.""" - t = P.TIKTOK - assert set(t.scopes_for("read")) < set(t.scopes_for("draft")) < set(t.scopes_for("post")) - assert t.default_capability == "post" - # video.publish is the whole difference between "we drafted it for you" and "we posted it". - assert "video.publish" not in t.scopes_for("draft") - assert "video.publish" in t.scopes_for("post") - - -# ---- per-provider consent params --------------------------------------------------------- -async def test_google_keeps_offline_consent_params(clients: AsyncClient, all_apps): - """access_type=offline + prompt=consent is what guarantees Google returns a refresh_token.""" - q = _q((await clients.post("/oauth/start", json={"provider": "google-search-console"})).json()) - assert q["access_type"] == ["offline"] and q["prompt"] == ["consent"] - - -def test_slack_is_bring_your_own_bot(): - """A Slack bot is workspace-scoped and belongs to the workspace it's installed in. A shared - treg app would sit between a team and their own messages — and couldn't be installed on their - behalf anyway — so the user brings their own token instead of consenting to ours.""" - assert P.SLACK.auth_kind == "token" - assert P.SLACK.scopes == {}, "no consent screen means no capability sizing" - assert P.SLACK.default_capability == "", "and nothing to default to" - - -async def test_each_provider_uses_its_own_client_credentials(clients: AsyncClient, all_apps): - for service, expected in (("google-search-console", "google-cid"), - ("google-tag-manager", "google-cid"), ("x", "x-cid")): - q = _q((await clients.post("/oauth/start", json={"provider": service})).json()) - assert q["client_id"] == [expected], service - - -# ---- scope gap detection (the re-consent trigger) ----------------------------------------- -def test_satisfied_capabilities_detects_a_scope_gap(): - """Providers never backfill scopes onto an issued grant — a later capability needs re-consent, - and this is how we know to prompt instead of letting the call 403.""" - gsc = P.GOOGLE_SEARCH_CONSOLE - read_only = gsc.scopes_for("read") - assert gsc.satisfied_capabilities(read_only) == ["read"] - assert "write" not in gsc.satisfied_capabilities(read_only) - both = read_only + gsc.scopes_for("write") - assert set(gsc.satisfied_capabilities(both)) == {"read", "write"} - - -def test_google_tag_manager_capabilities_are_cumulative_and_exclude_admin(): - """GTM can audit, prepare and publish without authority to delete an entire container or - administer the account's users. Each wider tier must still satisfy every narrower tier.""" - gtm = P.GOOGLE_TAG_MANAGER - assert set(gtm.scopes_for("read")) < set(gtm.scopes_for("write")) < set(gtm.scopes_for("manage")) - assert gtm.default_capability == "manage" - requested = {scope for scopes in gtm.scopes.values() for scope in scopes} - assert not { - "https://www.googleapis.com/auth/tagmanager.delete.containers", - "https://www.googleapis.com/auth/tagmanager.manage.users", - "https://www.googleapis.com/auth/tagmanager.manage.accounts", - } & requested - assert gtm.probe_path == gtm.discover_path == "/tagmanager/v2/accounts" - assert gtm.discover_key == "account" - assert gtm.discover_id_field == "path" - assert gtm.discover_label_field == "name" - - -async def test_unconfigured_providers_are_listed_but_flagged(clients: AsyncClient, monkeypatch): - monkeypatch.setenv("TREG_X_CLIENT_ID", "") - monkeypatch.setenv("TREG_X_CLIENT_SECRET", "") - get_settings.cache_clear() - try: - rows = {p["service"]: p for p in (await clients.get("/oauth/providers")).json()} - assert rows["x"]["configured"] is False - r = await clients.post("/oauth/start", json={"provider": "x"}) - assert r.status_code == 422 and "not configured" in r.text - # a bring-your-own-token provider needs nothing from the deployment, so it stays offerable - assert rows["slack"]["configured"] is True - finally: - get_settings.cache_clear() - - -# ---- the gap, surfaced on the connection -------------------------------------------------- -async def test_connection_reports_the_capability_it_lacks(clients: AsyncClient, all_apps): - """Connect read-only, then see that `write` is named as missing — the reconnect trigger.""" - body = { - "provider": "google-search-console", "capability": "read", - "token_uri": "http://upstream/token", # the in-process upstream stands in for Google - } - state = (await clients.post("/oauth/start", json=body)).json()["state"] - await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") - sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] - - conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] - assert conn["capabilities"] == ["read"] - assert conn["missing_capabilities"] == ["write"] - - -async def test_byo_connection_has_no_capability_fields(clients: AsyncClient): - body = {"name": "byo", "client_id": "c", "client_secret": "s", - "auth_uri": "http://p/auth", "token_uri": "http://upstream/token", "scopes": ["x"]} - state = (await clients.post("/oauth/start", json=body)).json()["state"] - await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") - sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] - conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] - assert "missing_capabilities" not in conn # nothing to compare against without a provider - - -# ---- LinkedIn ----------------------------------------------------------------------------- -def test_linkedin_has_one_capability(): - """These scopes let a member read their own profile and post as themselves. A read-only - LinkedIn connection could do nothing but identify you, so there is no second option worth - asking about — and a dialog with one real choice is just friction.""" - assert P.LINKEDIN.capabilities == ["write"] - assert "w_member_social" in P.LINKEDIN.scopes_for("write") - - -async def test_linkedin_does_not_get_googles_consent_params(clients: AsyncClient, monkeypatch): - monkeypatch.setenv("TREG_LINKEDIN_CLIENT_ID", "li-cid") - monkeypatch.setenv("TREG_LINKEDIN_CLIENT_SECRET", "li-csec") - get_settings.cache_clear() - try: - d = (await clients.post("/oauth/start", json={"provider": "linkedin"})).json() - q = _q(d) - assert q["client_id"] == ["li-cid"] - assert "access_type" not in q and "prompt" not in q, "LinkedIn rejects Google's params" - assert "w_member_social" in q["scope"][0] - finally: - get_settings.cache_clear() - - -def test_instagram_direct_and_page_grants_use_explicit_app_profiles(): - assert P.INSTAGRAM.client_id_setting == "instagram_client_id" - parsed = urlsplit(P.INSTAGRAM.base_url) - assert parsed.scheme == "https" - assert parsed.hostname == "graph.instagram.com" - page = P.INSTAGRAM.profile_for_authorization("facebook-page") - assert page.client_id_setting == P.FACEBOOK.client_id_setting == "meta_client_id" - assert page.base_url == P.FACEBOOK.base_url - - -def test_meta_capabilities_are_cumulative(): - """satisfied_capabilities() is set containment, so a non-cumulative tier would report a - connection that can publish but 'cannot read' — and the default capability would be wrong. - default_capability is the BROADEST tier by design (one honest consent screen beats - connect-twice), so adding manage moved the default there.""" - for provider in (P.FACEBOOK, P.INSTAGRAM): - assert set(provider.scopes["read"]) < set(provider.scopes["post"]), provider.service - assert set(provider.scopes["post"]) < set(provider.scopes["manage"]), provider.service - assert provider.default_capability == "manage", provider.service - - -def test_meta_messaging_stays_out_of_the_publish_tier(): - """A publish-only connect must never put "manage your messages" (or lead retrieval, or the - Page's Messenger inbox) on the consent screen — the two-way surfaces live only in manage.""" - two_way = { - "instagram_manage_messages", "instagram_manage_comments", "pages_messaging", - "pages_manage_engagement", "leads_retrieval", "catalog_management", - } - for provider in (P.FACEBOOK, P.INSTAGRAM): - for cap in ("read", "post"): - assert not two_way & set(provider.scopes[cap]), (provider.service, cap) - assert "instagram_business_manage_messages" in P.INSTAGRAM.scopes["manage"] - assert {"instagram_manage_messages", "pages_messaging"} <= set(P.INSTAGRAM.scopes["page-tools"]) - - -def test_lead_retrieval_brings_its_required_rider(): - """Meta only honors leads_retrieval alongside pages_manage_ads — requesting one without the - other consents fine and then 400s on /leads, which would demo as a broken integration.""" - manage = set(P.FACEBOOK.scopes["manage"]) - assert {"leads_retrieval", "pages_manage_ads"} <= manage - - -def test_instagram_login_is_direct_and_page_discovery_is_optional(): - for cap in ("read", "post", "manage"): - assert "pages_show_list" not in P.INSTAGRAM.scopes[cap] - assert P.INSTAGRAM.identity_required is True - page = P.INSTAGRAM.profile_for_authorization("facebook-page") - assert page.discover_id_field == "instagram_business_account.id" - assert "pages_show_list" in P.INSTAGRAM.scopes["page-tools"] - - -def test_meta_asks_for_a_long_lived_token(): - """Meta's code exchange yields a ~1-2h token and no refresh_token. Without the second - exchange every Meta connection dies the day it is made.""" - assert P.FACEBOOK.long_lived_exchange - assert P.INSTAGRAM.long_lived_exchange_style == "instagram" - assert not P.TIKTOK.long_lived_exchange # nothing else should have picked it up - - -def test_instagram_consent_never_mentions_page_publishing(): - """Scopes are per capability. An Instagram connect asking for pages_manage_posts would put - 'manage your Pages' posts' on the consent screen for authority it never uses.""" - for cap in P.INSTAGRAM.scopes.values(): - assert "pages_manage_posts" not in cap - - -def test_meta_page_discovery_can_walk_the_business_graph(): - """Most agency-held Pages (and the Instagram accounts linked to them) are OWNED by a Business - portfolio, where the member has business-level access and no personal Page role. Drop - business_management from either capability and that user consents cleanly, then gets an empty - picker — the extra listing 400s and is (rightly) swallowed.""" - for cap in P.FACEBOOK.scopes.values(): - assert "business_management" in cap - page = P.INSTAGRAM.profile_for_authorization("facebook-page") - assert "business_management" in P.INSTAGRAM.scopes["page-tools"] - for provider in (P.FACEBOOK, page): - assert provider.discover_extra_path.startswith("/me/businesses"), provider.service - assert provider.discover_extra_list_paths, provider.service - - -def test_meta_ads_needs_no_second_credential(): - """Google Ads is gated on a developer token from an approved manager account; Meta has no - equivalent, so a Meta Ads connect must yield a callable tool on its own.""" - assert P.META_ADS.can_autoprovision is True - assert P.META_ADS.needs_extra_credential is False - - -def test_meta_ads_read_can_still_list_accounts(): - """/me/adaccounts is a Business asset listing. Drop business_management from read and the - connect consents cleanly, then offers an empty account picker.""" - for cap in P.META_ADS.scopes.values(): - assert "business_management" in cap - assert set(P.META_ADS.scopes["read"]) < set(P.META_ADS.scopes["manage"]) - assert "ads_management" not in P.META_ADS.scopes["read"], "read must not be able to spend money" From 3314251934dd28a1832fab9e30c941e80574fb5d Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 17:06:33 +0200 Subject: [PATCH 23/31] feedjolt: restore own_account GETs now that REST dual-auth is live --- tests/test_oauth_providers_m3.py | 308 +++++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) diff --git a/tests/test_oauth_providers_m3.py b/tests/test_oauth_providers_m3.py index ace59e933..cda678cd6 100644 --- a/tests/test_oauth_providers_m3.py +++ b/tests/test_oauth_providers_m3.py @@ -57,3 +57,311 @@ def test_every_provider_is_registered(): "coingecko", "polygon", "finnhub", "twelvedata", "fmp", "eodhd", "marketstack", "tiingo", "microsoft-ads", "snapchat-ads", "tiktok-ads", "pinterest-ads", } + + +def test_default_capability_is_the_broadest(): + """Connect asks for the fullest capability; a narrower one is chosen up front, not bolted on + afterwards. Every provider's write must be a superset of its read for that to be safe.""" + assert P.GOOGLE_SEARCH_CONSOLE.default_capability == "write" + assert P.X.default_capability == "write" + assert P.GOOGLE_ADS.default_capability == "manage" # it has no read-only mode + assert P.GOOGLE_TAG_MANAGER.default_capability == "manage" + for provider in P.REGISTRY.values(): + caps = provider.capabilities + if "read" in caps and "write" in caps: + assert set(provider.scopes["read"]) < set(provider.scopes["write"]), provider.service + + +def test_google_ads_refuses_to_autoprovision(): + """Ads needs a developer-token header too; a bearer-only tool would 401 on first use.""" + assert P.GOOGLE_ADS.can_autoprovision is False + assert "developer-token" in P.GOOGLE_ADS.extra_credential_note + assert P.GOOGLE_SEARCH_CONSOLE.can_autoprovision is True + + +def test_x_write_keeps_offline_access(): + """Without offline.access the token can't be refreshed and every X connection becomes a + manual-reconnect chore within hours.""" + assert "offline.access" in P.X.scopes_for("write") + assert "offline.access" in P.X.scopes_for("read") + + +# ---- X's two quirks ---------------------------------------------------------------------- +async def test_x_consent_url_carries_a_pkce_challenge(clients: AsyncClient, all_apps): + d = (await clients.post("/oauth/start", json={"provider": "x"})).json() + q = _q(d) + assert q["code_challenge_method"] == ["S256"] + assert q["code_challenge"], "X rejects a code exchanged without a verifier" + # the verifier itself must stay server-side + assert "code_verifier" not in q + + +async def test_pkce_challenge_matches_the_stored_verifier(clients: AsyncClient, all_apps): + d = (await clients.post("/oauth/start", json={"provider": "x"})).json() + challenge = _q(d)["code_challenge"][0] + async with session_maker() as db: + p = (await db.execute(select(PendingOAuth).where(PendingOAuth.state == d["state"]))).scalars().one() + assert p.code_verifier + assert oauth.pkce_challenge(p.code_verifier) == challenge + assert p.token_endpoint_auth_method == "client_secret_basic" + + +async def test_google_does_not_use_pkce(clients: AsyncClient, all_apps): + d = (await clients.post("/oauth/start", json={"provider": "google-search-console"})).json() + assert "code_challenge" not in _q(d) + + +# ---- TikTok's two quirks ------------------------------------------------------------------- +async def test_tiktok_consent_url_uses_client_key_not_client_id(clients: AsyncClient, all_apps): + """TikTok ignores the OAuth2 spelling. Sending `client_id` gets a consent page that errors out + rather than an obvious 400, so this is worth pinning.""" + q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) + assert q["client_key"] == ["tiktok-cid"] + assert "client_id" not in q + + +async def test_tiktok_comma_joins_its_scopes(clients: AsyncClient, all_apps): + """Space-joined scopes come back from TikTok as scope_not_authorized — it splits on commas.""" + q = _q((await clients.post("/oauth/start", json={"provider": "tiktok"})).json()) + scope = q["scope"][0] + assert "," in scope and " " not in scope + assert set(scope.split(",")) == set(P.TIKTOK.scopes_for(P.TIKTOK.default_capability)) + + +async def test_tiktok_granted_scopes_are_stored_space_joined(clients: AsyncClient, all_apps, monkeypatch): + """The wire dialect must not leak into storage: every reader of granted_scopes uses .split(), + so a comma-joined grant would read as one bogus scope and report every capability unsatisfied.""" + # Registry mode takes token_uri from the provider, not the body, so point the provider itself at + # the in-process upstream (frozen dataclass → replace rather than setattr). + monkeypatch.setitem(P.REGISTRY, "tiktok", replace(P.TIKTOK, token_uri="http://upstream/token")) + body = {"provider": "tiktok", "capability": "post"} + state = (await clients.post("/oauth/start", json=body)).json()["state"] + await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") + sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] + + conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] + assert set(conn["capabilities"]) == {"read", "draft", "post"} + + +def test_tiktok_capabilities_are_cumulative(): + """draft must contain read and post must contain draft, or satisfied_capabilities() (which is + set-containment) reports a connection that can post but cannot read.""" + t = P.TIKTOK + assert set(t.scopes_for("read")) < set(t.scopes_for("draft")) < set(t.scopes_for("post")) + assert t.default_capability == "post" + # video.publish is the whole difference between "we drafted it for you" and "we posted it". + assert "video.publish" not in t.scopes_for("draft") + assert "video.publish" in t.scopes_for("post") + + +# ---- per-provider consent params --------------------------------------------------------- +async def test_google_keeps_offline_consent_params(clients: AsyncClient, all_apps): + """access_type=offline + prompt=consent is what guarantees Google returns a refresh_token.""" + q = _q((await clients.post("/oauth/start", json={"provider": "google-search-console"})).json()) + assert q["access_type"] == ["offline"] and q["prompt"] == ["consent"] + + +def test_slack_is_bring_your_own_bot(): + """A Slack bot is workspace-scoped and belongs to the workspace it's installed in. A shared + treg app would sit between a team and their own messages — and couldn't be installed on their + behalf anyway — so the user brings their own token instead of consenting to ours.""" + assert P.SLACK.auth_kind == "token" + assert P.SLACK.scopes == {}, "no consent screen means no capability sizing" + assert P.SLACK.default_capability == "", "and nothing to default to" + + +async def test_each_provider_uses_its_own_client_credentials(clients: AsyncClient, all_apps): + for service, expected in (("google-search-console", "google-cid"), + ("google-tag-manager", "google-cid"), ("x", "x-cid")): + q = _q((await clients.post("/oauth/start", json={"provider": service})).json()) + assert q["client_id"] == [expected], service + + +# ---- scope gap detection (the re-consent trigger) ----------------------------------------- +def test_satisfied_capabilities_detects_a_scope_gap(): + """Providers never backfill scopes onto an issued grant — a later capability needs re-consent, + and this is how we know to prompt instead of letting the call 403.""" + gsc = P.GOOGLE_SEARCH_CONSOLE + read_only = gsc.scopes_for("read") + assert gsc.satisfied_capabilities(read_only) == ["read"] + assert "write" not in gsc.satisfied_capabilities(read_only) + both = read_only + gsc.scopes_for("write") + assert set(gsc.satisfied_capabilities(both)) == {"read", "write"} + + +def test_google_tag_manager_capabilities_are_cumulative_and_exclude_admin(): + """GTM can audit, prepare and publish without authority to delete an entire container or + administer the account's users. Each wider tier must still satisfy every narrower tier.""" + gtm = P.GOOGLE_TAG_MANAGER + assert set(gtm.scopes_for("read")) < set(gtm.scopes_for("write")) < set(gtm.scopes_for("manage")) + assert gtm.default_capability == "manage" + requested = {scope for scopes in gtm.scopes.values() for scope in scopes} + assert not { + "https://www.googleapis.com/auth/tagmanager.delete.containers", + "https://www.googleapis.com/auth/tagmanager.manage.users", + "https://www.googleapis.com/auth/tagmanager.manage.accounts", + } & requested + assert gtm.probe_path == gtm.discover_path == "/tagmanager/v2/accounts" + assert gtm.discover_key == "account" + assert gtm.discover_id_field == "path" + assert gtm.discover_label_field == "name" + + +async def test_unconfigured_providers_are_listed_but_flagged(clients: AsyncClient, monkeypatch): + monkeypatch.setenv("TREG_X_CLIENT_ID", "") + monkeypatch.setenv("TREG_X_CLIENT_SECRET", "") + get_settings.cache_clear() + try: + rows = {p["service"]: p for p in (await clients.get("/oauth/providers")).json()} + assert rows["x"]["configured"] is False + r = await clients.post("/oauth/start", json={"provider": "x"}) + assert r.status_code == 422 and "not configured" in r.text + # a bring-your-own-token provider needs nothing from the deployment, so it stays offerable + assert rows["slack"]["configured"] is True + finally: + get_settings.cache_clear() + + +# ---- the gap, surfaced on the connection -------------------------------------------------- +async def test_connection_reports_the_capability_it_lacks(clients: AsyncClient, all_apps): + """Connect read-only, then see that `write` is named as missing — the reconnect trigger.""" + body = { + "provider": "google-search-console", "capability": "read", + "token_uri": "http://upstream/token", # the in-process upstream stands in for Google + } + state = (await clients.post("/oauth/start", json=body)).json()["state"] + await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") + sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] + + conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] + assert conn["capabilities"] == ["read"] + assert conn["missing_capabilities"] == ["write"] + + +async def test_byo_connection_has_no_capability_fields(clients: AsyncClient): + body = {"name": "byo", "client_id": "c", "client_secret": "s", + "auth_uri": "http://p/auth", "token_uri": "http://upstream/token", "scopes": ["x"]} + state = (await clients.post("/oauth/start", json=body)).json()["state"] + await clients.get(f"/oauth/callback?code=AUTHCODE&state={state}") + sid = (await clients.get(f"/oauth/status/{state}")).json()["secret_id"] + conn = {c["id"]: c for c in (await clients.get("/connections")).json()}[sid] + assert "missing_capabilities" not in conn # nothing to compare against without a provider + + +# ---- LinkedIn ----------------------------------------------------------------------------- +def test_linkedin_has_one_capability(): + """These scopes let a member read their own profile and post as themselves. A read-only + LinkedIn connection could do nothing but identify you, so there is no second option worth + asking about — and a dialog with one real choice is just friction.""" + assert P.LINKEDIN.capabilities == ["write"] + assert "w_member_social" in P.LINKEDIN.scopes_for("write") + + +async def test_linkedin_does_not_get_googles_consent_params(clients: AsyncClient, monkeypatch): + monkeypatch.setenv("TREG_LINKEDIN_CLIENT_ID", "li-cid") + monkeypatch.setenv("TREG_LINKEDIN_CLIENT_SECRET", "li-csec") + get_settings.cache_clear() + try: + d = (await clients.post("/oauth/start", json={"provider": "linkedin"})).json() + q = _q(d) + assert q["client_id"] == ["li-cid"] + assert "access_type" not in q and "prompt" not in q, "LinkedIn rejects Google's params" + assert "w_member_social" in q["scope"][0] + finally: + get_settings.cache_clear() + + +def test_instagram_direct_and_page_grants_use_explicit_app_profiles(): + assert P.INSTAGRAM.client_id_setting == "instagram_client_id" + parsed = urlsplit(P.INSTAGRAM.base_url) + assert parsed.scheme == "https" + assert parsed.hostname == "graph.instagram.com" + page = P.INSTAGRAM.profile_for_authorization("facebook-page") + assert page.client_id_setting == P.FACEBOOK.client_id_setting == "meta_client_id" + assert page.base_url == P.FACEBOOK.base_url + + +def test_meta_capabilities_are_cumulative(): + """satisfied_capabilities() is set containment, so a non-cumulative tier would report a + connection that can publish but 'cannot read' — and the default capability would be wrong. + default_capability is the BROADEST tier by design (one honest consent screen beats + connect-twice), so adding manage moved the default there.""" + for provider in (P.FACEBOOK, P.INSTAGRAM): + assert set(provider.scopes["read"]) < set(provider.scopes["post"]), provider.service + assert set(provider.scopes["post"]) < set(provider.scopes["manage"]), provider.service + assert provider.default_capability == "manage", provider.service + + +def test_meta_messaging_stays_out_of_the_publish_tier(): + """A publish-only connect must never put "manage your messages" (or lead retrieval, or the + Page's Messenger inbox) on the consent screen — the two-way surfaces live only in manage.""" + two_way = { + "instagram_manage_messages", "instagram_manage_comments", "pages_messaging", + "pages_manage_engagement", "leads_retrieval", "catalog_management", + } + for provider in (P.FACEBOOK, P.INSTAGRAM): + for cap in ("read", "post"): + assert not two_way & set(provider.scopes[cap]), (provider.service, cap) + assert "instagram_business_manage_messages" in P.INSTAGRAM.scopes["manage"] + assert {"instagram_manage_messages", "pages_messaging"} <= set(P.INSTAGRAM.scopes["page-tools"]) + + +def test_lead_retrieval_brings_its_required_rider(): + """Meta only honors leads_retrieval alongside pages_manage_ads — requesting one without the + other consents fine and then 400s on /leads, which would demo as a broken integration.""" + manage = set(P.FACEBOOK.scopes["manage"]) + assert {"leads_retrieval", "pages_manage_ads"} <= manage + + +def test_instagram_login_is_direct_and_page_discovery_is_optional(): + for cap in ("read", "post", "manage"): + assert "pages_show_list" not in P.INSTAGRAM.scopes[cap] + assert P.INSTAGRAM.identity_required is True + page = P.INSTAGRAM.profile_for_authorization("facebook-page") + assert page.discover_id_field == "instagram_business_account.id" + assert "pages_show_list" in P.INSTAGRAM.scopes["page-tools"] + + +def test_meta_asks_for_a_long_lived_token(): + """Meta's code exchange yields a ~1-2h token and no refresh_token. Without the second + exchange every Meta connection dies the day it is made.""" + assert P.FACEBOOK.long_lived_exchange + assert P.INSTAGRAM.long_lived_exchange_style == "instagram" + assert not P.TIKTOK.long_lived_exchange # nothing else should have picked it up + + +def test_instagram_consent_never_mentions_page_publishing(): + """Scopes are per capability. An Instagram connect asking for pages_manage_posts would put + 'manage your Pages' posts' on the consent screen for authority it never uses.""" + for cap in P.INSTAGRAM.scopes.values(): + assert "pages_manage_posts" not in cap + + +def test_meta_page_discovery_can_walk_the_business_graph(): + """Most agency-held Pages (and the Instagram accounts linked to them) are OWNED by a Business + portfolio, where the member has business-level access and no personal Page role. Drop + business_management from either capability and that user consents cleanly, then gets an empty + picker — the extra listing 400s and is (rightly) swallowed.""" + for cap in P.FACEBOOK.scopes.values(): + assert "business_management" in cap + page = P.INSTAGRAM.profile_for_authorization("facebook-page") + assert "business_management" in P.INSTAGRAM.scopes["page-tools"] + for provider in (P.FACEBOOK, page): + assert provider.discover_extra_path.startswith("/me/businesses"), provider.service + assert provider.discover_extra_list_paths, provider.service + + +def test_meta_ads_needs_no_second_credential(): + """Google Ads is gated on a developer token from an approved manager account; Meta has no + equivalent, so a Meta Ads connect must yield a callable tool on its own.""" + assert P.META_ADS.can_autoprovision is True + assert P.META_ADS.needs_extra_credential is False + + +def test_meta_ads_read_can_still_list_accounts(): + """/me/adaccounts is a Business asset listing. Drop business_management from read and the + connect consents cleanly, then offers an empty account picker.""" + for cap in P.META_ADS.scopes.values(): + assert "business_management" in cap + assert set(P.META_ADS.scopes["read"]) < set(P.META_ADS.scopes["manage"]) + assert "ads_management" not in P.META_ADS.scopes["read"], "read must not be able to spend money" From 496f0fc41fd4bce817d85f0347c329bc731e65d3 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 17:06:56 +0200 Subject: [PATCH 24/31] feedjolt: restore own_account GETs now that REST dual-auth is live --- render.yaml | 258 +--------------------------------------------------- 1 file changed, 2 insertions(+), 256 deletions(-) diff --git a/render.yaml b/render.yaml index 5131784dd..3d0fa2562 100644 --- a/render.yaml +++ b/render.yaml @@ -97,261 +97,7 @@ services: sync: false - key: TREG_PLATFORM_KEY_HUNTER sync: false - - key: TREG_PLATFORM_KEY_LEADMAGIC - sync: false - - key: TREG_PLATFORM_KEY_LUSHA - sync: false - - key: TREG_PLATFORM_KEY_PDL - sync: false - - key: TREG_PLATFORM_KEY_DIFFBOT - sync: false - - key: TREG_PLATFORM_KEY_AKTA - sync: false - - key: TREG_PLATFORM_KEY_APIFY - sync: false - - key: TREG_PLATFORM_KEY_SERPSTAT # Serpstat API token (?token=…) - sync: false - - key: TREG_PLATFORM_KEY_SPYFU # SpyFu SECRET KEY alone (?api_key=…), not id/base64 - sync: false - - key: TREG_PLATFORM_KEY_CORESIGNAL # Coresignal API key (`apikey` header) - sync: false - - key: TREG_PLATFORM_KEY_THECOMPANIESAPI # raw token, injected as "Basic {secret}" un-encoded - sync: false - - key: TREG_PLATFORM_KEY_COMPANYENRICH # CompanyEnrich API key (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_OCEANIO # Ocean.io API token (X-Api-Token); tier 4 refused until fx.yaml prices it - sync: false - - key: TREG_PLATFORM_KEY_PREDICTLEADS # base64 of "api_key:api_token" (HTTP Basic) - sync: false - - key: TREG_PLATFORM_KEY_FINDYMAIL # Findymail API key (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_BRANDDEV # Brand.dev API key (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_ICYPEAS # Icypeas API key (raw Authorization header) - sync: false - - key: TREG_PLATFORM_KEY_LEADSFORGE # LeadsForge API key (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_INFLUENCERSCLUB # influencers.club dashboard API key, a JWT (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_CRUSTDATA # Crustdata API key; x-api-version is injected from provider metadata - sync: false - - key: TREG_PLATFORM_KEY_AVIATO # Aviato API key (Authorization: Bearer) - sync: false - - key: TREG_PLATFORM_KEY_EXA # Exa API key (x-api-key); every response reports costDollars - sync: false - - key: TREG_PLATFORM_KEY_FIBER_AI # Fiber AI key (x-api-key) + - key: TREG_PLATFORM_KEY_FEEDJOLT # plan-included; BYOK until a per-call rate card exists sync: false - - key: TREG_PLATFORM_KEY_TOMBA # Tomba API key ta_… (X-Tomba-Key) - sync: false - - key: TREG_PLATFORM_KEY_TOMBA_SECRET # Tomba API secret ts_… (X-Tomba-Secret) — set BOTH - sync: false - - key: TREG_PLATFORM_PROVIDERS # e.g. "tikhub,dataforseo,scrapecreators"; "" = tier 4 off - value: "" - # Overflow (docs/context/ops/capacity.md): treg-owned aggregator accounts serving the SAME vendor - # endpoint when our own account is out. MODE is the switch: off (default) | shadow | on. - - key: TREG_OVERFLOW_MODE - value: "off" - - key: TREG_OVERFLOW_DAILY_BUDGET_USD - value: "20" - - key: TREG_OVERFLOW_KEY_ORTHOGONAL - sync: false - - key: TREG_OVERFLOW_KEY_MONID - sync: false - # Worker profile: the hourly capacity sweep (docs/context/ops/capacity.md). Same code, same env - # as the web service (fromService — add a platform key in the web service only), no HTTP. - - type: cron - name: treg-capacity-sweep - runtime: python - region: oregon - plan: starter - branch: main - schedule: "17 * * * *" - buildCommand: pip install ".[server]" - startCommand: treg-worker capacity sweep - envVars: - - key: TREG_DATABASE_URL - fromDatabase: - name: treg-db - property: connectionString - - key: PYTHON_VERSION - value: "3.12.7" - - key: TREG_SECRET_KEY - fromService: - type: web - name: treg - envVarName: TREG_SECRET_KEY - - key: TREG_OVERFLOW_KEY_ORTHOGONAL - fromService: - type: web - name: treg - envVarName: TREG_OVERFLOW_KEY_ORTHOGONAL - - key: TREG_OVERFLOW_KEY_MONID - fromService: - type: web - name: treg - envVarName: TREG_OVERFLOW_KEY_MONID - - key: TREG_PLATFORM_KEY_TIKHUB - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_TIKHUB - - key: TREG_PLATFORM_KEY_DATAFORSEO - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_DATAFORSEO - - key: TREG_PLATFORM_KEY_SCRAPECREATORS - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_SCRAPECREATORS - - key: TREG_PLATFORM_KEY_BRIGHTDATA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_BRIGHTDATA - - key: TREG_PLATFORM_KEY_JUSTONEAPI - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_JUSTONEAPI - - key: TREG_PLATFORM_KEY_SERPAPI - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_SERPAPI - - key: TREG_PLATFORM_KEY_MOZ - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_MOZ - - key: TREG_PLATFORM_KEY_SERANKING - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_SERANKING - - key: TREG_PLATFORM_KEY_HUNTER - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_HUNTER - key: TREG_PLATFORM_KEY_LEADMAGIC - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_LEADMAGIC - - key: TREG_PLATFORM_KEY_LUSHA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_LUSHA - - key: TREG_PLATFORM_KEY_PDL - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_PDL - - key: TREG_PLATFORM_KEY_DIFFBOT - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_DIFFBOT - - key: TREG_PLATFORM_KEY_AKTA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_AKTA - - key: TREG_PLATFORM_KEY_APIFY - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_APIFY - - key: TREG_PLATFORM_KEY_SERPSTAT - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_SERPSTAT - - key: TREG_PLATFORM_KEY_SPYFU - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_SPYFU - - key: TREG_PLATFORM_KEY_CORESIGNAL - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_CORESIGNAL - - key: TREG_PLATFORM_KEY_THECOMPANIESAPI - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_THECOMPANIESAPI - - key: TREG_PLATFORM_KEY_COMPANYENRICH - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_COMPANYENRICH - - key: TREG_PLATFORM_KEY_OCEANIO - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_OCEANIO - - key: TREG_PLATFORM_KEY_PREDICTLEADS - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_PREDICTLEADS - - key: TREG_PLATFORM_KEY_FINDYMAIL - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_FINDYMAIL - - key: TREG_PLATFORM_KEY_BRANDDEV - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_BRANDDEV - - key: TREG_PLATFORM_KEY_ICYPEAS - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_ICYPEAS - - key: TREG_PLATFORM_KEY_LEADSFORGE - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_LEADSFORGE - - key: TREG_PLATFORM_KEY_INFLUENCERSCLUB - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_INFLUENCERSCLUB - - key: TREG_PLATFORM_KEY_CRUSTDATA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_CRUSTDATA - - key: TREG_PLATFORM_KEY_AVIATO - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_AVIATO - - key: TREG_PLATFORM_KEY_EXA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_EXA - - key: TREG_PLATFORM_KEY_FIBER_AI - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_FIBER_AI - - key: TREG_PLATFORM_KEY_TOMBA - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_TOMBA - - key: TREG_PLATFORM_KEY_TOMBA_SECRET - fromService: - type: web - name: treg - envVarName: TREG_PLATFORM_KEY_TOMBA_SECRET + sync: false From d4b8120d8d87e4ff3fa13354ac568b96f75bc186 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 17:07:45 +0200 Subject: [PATCH 25/31] feedjolt: restore own_account GETs now that REST dual-auth is live --- render.yaml | 256 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) diff --git a/render.yaml b/render.yaml index 3d0fa2562..36ffb0aef 100644 --- a/render.yaml +++ b/render.yaml @@ -101,3 +101,259 @@ services: sync: false - key: TREG_PLATFORM_KEY_LEADMAGIC sync: false + - key: TREG_PLATFORM_KEY_LUSHA + sync: false + - key: TREG_PLATFORM_KEY_PDL + sync: false + - key: TREG_PLATFORM_KEY_DIFFBOT + sync: false + - key: TREG_PLATFORM_KEY_AKTA + sync: false + - key: TREG_PLATFORM_KEY_APIFY + sync: false + - key: TREG_PLATFORM_KEY_SERPSTAT # Serpstat API token (?token=…) + sync: false + - key: TREG_PLATFORM_KEY_SPYFU # SpyFu SECRET KEY alone (?api_key=…), not id/base64 + sync: false + - key: TREG_PLATFORM_KEY_CORESIGNAL # Coresignal API key (`apikey` header) + sync: false + - key: TREG_PLATFORM_KEY_THECOMPANIESAPI # raw token, injected as "Basic {secret}" un-encoded + sync: false + - key: TREG_PLATFORM_KEY_COMPANYENRICH # CompanyEnrich API key (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_OCEANIO # Ocean.io API token (X-Api-Token); tier 4 refused until fx.yaml prices it + sync: false + - key: TREG_PLATFORM_KEY_PREDICTLEADS # base64 of "api_key:api_token" (HTTP Basic) + sync: false + - key: TREG_PLATFORM_KEY_FINDYMAIL # Findymail API key (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_BRANDDEV # Brand.dev API key (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_ICYPEAS # Icypeas API key (raw Authorization header) + sync: false + - key: TREG_PLATFORM_KEY_LEADSFORGE # LeadsForge API key (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_INFLUENCERSCLUB # influencers.club dashboard API key, a JWT (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_CRUSTDATA # Crustdata API key; x-api-version is injected from provider metadata + sync: false + - key: TREG_PLATFORM_KEY_AVIATO # Aviato API key (Authorization: Bearer) + sync: false + - key: TREG_PLATFORM_KEY_EXA # Exa API key (x-api-key); every response reports costDollars + sync: false + - key: TREG_PLATFORM_KEY_FIBER_AI # Fiber AI key (x-api-key) + sync: false + - key: TREG_PLATFORM_KEY_TOMBA # Tomba API key ta_… (X-Tomba-Key) + sync: false + - key: TREG_PLATFORM_KEY_TOMBA_SECRET # Tomba API secret ts_… (X-Tomba-Secret) — set BOTH + sync: false + - key: TREG_PLATFORM_PROVIDERS # e.g. "tikhub,dataforseo,scrapecreators"; "" = tier 4 off + value: "" + # Overflow (docs/context/ops/capacity.md): treg-owned aggregator accounts serving the SAME vendor + # endpoint when our own account is out. MODE is the switch: off (default) | shadow | on. + - key: TREG_OVERFLOW_MODE + value: "off" + - key: TREG_OVERFLOW_DAILY_BUDGET_USD + value: "20" + - key: TREG_OVERFLOW_KEY_ORTHOGONAL + sync: false + - key: TREG_OVERFLOW_KEY_MONID + sync: false + # Worker profile: the hourly capacity sweep (docs/context/ops/capacity.md). Same code, same env + # as the web service (fromService — add a platform key in the web service only), no HTTP. + - type: cron + name: treg-capacity-sweep + runtime: python + region: oregon + plan: starter + branch: main + schedule: "17 * * * *" + buildCommand: pip install ".[server]" + startCommand: treg-worker capacity sweep + envVars: + - key: TREG_DATABASE_URL + fromDatabase: + name: treg-db + property: connectionString + - key: PYTHON_VERSION + value: "3.12.7" + - key: TREG_SECRET_KEY + fromService: + type: web + name: treg + envVarName: TREG_SECRET_KEY + - key: TREG_OVERFLOW_KEY_ORTHOGONAL + fromService: + type: web + name: treg + envVarName: TREG_OVERFLOW_KEY_ORTHOGONAL + - key: TREG_OVERFLOW_KEY_MONID + fromService: + type: web + name: treg + envVarName: TREG_OVERFLOW_KEY_MONID + - key: TREG_PLATFORM_KEY_TIKHUB + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_TIKHUB + - key: TREG_PLATFORM_KEY_DATAFORSEO + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_DATAFORSEO + - key: TREG_PLATFORM_KEY_SCRAPECREATORS + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_SCRAPECREATORS + - key: TREG_PLATFORM_KEY_BRIGHTDATA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_BRIGHTDATA + - key: TREG_PLATFORM_KEY_JUSTONEAPI + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_JUSTONEAPI + - key: TREG_PLATFORM_KEY_SERPAPI + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_SERPAPI + - key: TREG_PLATFORM_KEY_MOZ + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_MOZ + - key: TREG_PLATFORM_KEY_SERANKING + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_SERANKING + - key: TREG_PLATFORM_KEY_HUNTER + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_HUNTER + - key: TREG_PLATFORM_KEY_LEADMAGIC + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_LEADMAGIC + - key: TREG_PLATFORM_KEY_LUSHA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_LUSHA + - key: TREG_PLATFORM_KEY_PDL + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_PDL + - key: TREG_PLATFORM_KEY_DIFFBOT + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_DIFFBOT + - key: TREG_PLATFORM_KEY_AKTA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_AKTA + - key: TREG_PLATFORM_KEY_APIFY + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_APIFY + - key: TREG_PLATFORM_KEY_SERPSTAT + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_SERPSTAT + - key: TREG_PLATFORM_KEY_SPYFU + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_SPYFU + - key: TREG_PLATFORM_KEY_CORESIGNAL + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_CORESIGNAL + - key: TREG_PLATFORM_KEY_THECOMPANIESAPI + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_THECOMPANIESAPI + - key: TREG_PLATFORM_KEY_COMPANYENRICH + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_COMPANYENRICH + - key: TREG_PLATFORM_KEY_OCEANIO + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_OCEANIO + - key: TREG_PLATFORM_KEY_PREDICTLEADS + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_PREDICTLEADS + - key: TREG_PLATFORM_KEY_FINDYMAIL + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_FINDYMAIL + - key: TREG_PLATFORM_KEY_BRANDDEV + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_BRANDDEV + - key: TREG_PLATFORM_KEY_ICYPEAS + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_ICYPEAS + - key: TREG_PLATFORM_KEY_LEADSFORGE + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_LEADSFORGE + - key: TREG_PLATFORM_KEY_INFLUENCERSCLUB + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_INFLUENCERSCLUB + - key: TREG_PLATFORM_KEY_CRUSTDATA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_CRUSTDATA + - key: TREG_PLATFORM_KEY_AVIATO + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_AVIATO + - key: TREG_PLATFORM_KEY_EXA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_EXA + - key: TREG_PLATFORM_KEY_FIBER_AI + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_FIBER_AI + - key: TREG_PLATFORM_KEY_TOMBA + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_TOMBA + - key: TREG_PLATFORM_KEY_TOMBA_SECRET + fromService: + type: web + name: treg + envVarName: TREG_PLATFORM_KEY_TOMBA_SECRET From 29794522b8651f5ac2f64b0c235fea1af1e06687 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 17:07:58 +0200 Subject: [PATCH 26/31] feedjolt: restore own_account GETs now that REST dual-auth is live --- src/treg/oauth_feedjolt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/treg/oauth_feedjolt.py b/src/treg/oauth_feedjolt.py index 245313ab3..e6ea9c22b 100644 --- a/src/treg/oauth_feedjolt.py +++ b/src/treg/oauth_feedjolt.py @@ -28,7 +28,7 @@ setup_note=( "REST calls are included in the plan, not billed per call. " "GET /workspaces is the probe. Do not use GET /health — it is unauthenticated. " - "A garbage Bearer or X-API-Key returns 401 {\"detail\":\"Invalid or revoked API key\"}." + "A garbage Bearer/X-API-Key returns 401 {\"detail\":\"Invalid or revoked API key\"}." ), auth_uri="", token_uri="", scopes={}, From beb4eb4ce93bf111cb47ba5c2ad8f280e9d73c53 Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 17:08:50 +0200 Subject: [PATCH 27/31] feedjolt: restore own_account GETs now that REST dual-auth is live --- src/treg/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/treg/config.py b/src/treg/config.py index c274de665..2773fa41c 100644 --- a/src/treg/config.py +++ b/src/treg/config.py @@ -149,6 +149,7 @@ def _async_pg_driver(cls, v: str) -> str: platform_key_moz: str = "" # base64 of "access_id:secret_key" (HTTP Basic) platform_key_seranking: str = "" platform_key_hunter: str = "" + platform_key_feedjolt: str = "" # Bearer fjk_…; plan-included REST, no per-call meter platform_key_leadmagic: str = "" platform_key_lusha: str = "" platform_key_pdl: str = "" From ca0884115eb48cf4e61a207bbc4d3cbc9ba89c4f Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 17:10:16 +0200 Subject: [PATCH 28/31] feedjolt: restore own_account GETs now that REST dual-auth is live --- src/treg/catalog/feedjolt.yaml | 417 +-------------------------------- 1 file changed, 1 insertion(+), 416 deletions(-) diff --git a/src/treg/catalog/feedjolt.yaml b/src/treg/catalog/feedjolt.yaml index 754a86b98..b711b2379 100644 --- a/src/treg/catalog/feedjolt.yaml +++ b/src/treg/catalog/feedjolt.yaml @@ -11,6 +11,7 @@ source: # treg relays REST HTTP, not JSON-RPC. # # Probe is GET /workspaces. GET /health is unauthenticated (200 with no key) and must not be used. +# Auth: Authorization Bearer fjk_ (X-API-Key also works). The oauth sidecar sends Bearer. limits: "plan-included; 429 on abuse. REST API is not on the Growth plan." pricing_url: https://www.feedjolt.com/en @@ -24,419 +25,3 @@ proposed_capabilities: product.feedback.changelog.list: "List published changelog entries" product.feedback.statuses.list: "List feedback statuses in a workspace" product.feedback.tags.list: "List tags in a feedback workspace" - -endpoints: - - id: feedjolt.public.boards.list - capability: product.feedback.boards.list - platform: web - scope: any_account - method: GET - path: /public/{workspace_slug}/boards - name: "List public boards" - summary: "List Public Boards" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - test_request: - pathParams: {workspace_slug: "corvidly"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Public portal read; no API key required." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.public.posts.list - capability: product.feedback.posts.list - platform: web - scope: any_account - method: GET - path: /public/{workspace_slug}/boards/{board_slug}/posts - name: "List public posts on a board" - summary: "List Public Posts" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - board_slug: {type: string, required: true, example: "feedback"} - queryParams: - page: {type: integer, required: false, example: 1} - page_size: {type: integer, required: false, example: 3} - sort_by: {type: string, required: false} - q: {type: string, required: false} - status_id: {type: string, required: false} - tag_ids: {type: string, required: false} - test_request: - pathParams: {workspace_slug: "corvidly", board_slug: "feedback"} - queryParams: {page_size: 3} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Public portal read; no API key required." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.public.posts.get - capability: product.feedback.posts.get - platform: web - scope: any_account - method: GET - path: /public/{workspace_slug}/posts/{post_id} - name: "Get a public post" - summary: "Get Public Post" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - post_id: {type: string, required: true, example: "52428c9b-971a-4804-a96e-73ce3da4ab17"} - test_request: - pathParams: {workspace_slug: "corvidly", post_id: "52428c9b-971a-4804-a96e-73ce3da4ab17"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Public portal read; no API key required." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.public.posts.search - capability: product.feedback.posts.search - platform: web - scope: any_account - method: GET - path: /public/{workspace_slug}/search - name: "Search public posts" - summary: "Search Public Posts" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - queryParams: - q: {type: string, required: true, example: "MCP", note: "minLength 2, maxLength 200"} - test_request: - pathParams: {workspace_slug: "corvidly"} - queryParams: {q: "MCP"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Public portal read. OpenAPI: q minLength 2, maxLength 200." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.public.roadmap.get - capability: product.feedback.roadmap.get - platform: web - scope: any_account - method: GET - path: /public/{workspace_slug}/roadmap - name: "Get public roadmap" - summary: "Get Public Roadmap" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - queryParams: - tags: {type: string, required: false, note: "comma-separated tag ids"} - test_request: - pathParams: {workspace_slug: "corvidly"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Public portal read." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.public.changelog.list - capability: product.feedback.changelog.list - platform: web - scope: any_account - method: GET - path: /public/{workspace_slug}/changelog - name: "List public changelog" - summary: "List Public Changelog" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - queryParams: - page: {type: integer, required: false, example: 1} - page_size: {type: integer, required: false, example: 5} - version_id: {type: string, required: false} - test_request: - pathParams: {workspace_slug: "corvidly"} - queryParams: {page_size: 5} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Public portal read." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.account.workspaces.list - kind: account - capability: account.usage - platform: account - scope: own_account - method: GET - path: /workspaces - name: "List workspaces for this API key" - summary: "List Workspaces" - input: - note: "no parameters — the key rides in the Authorization Bearer header (X-API-Key also works). This is the registry probe." - test_request: {queryParams: {}} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Probe. Garbage Bearer/X-API-Key returns 401 {\"detail\":\"Invalid or revoked API key\"}. Cookie-missing (no auth) returns 401 {\"detail\":\"Not authenticated\"}. Valid-key 2xx pending maintainer credential." - docs_url: https://www.feedjolt.com/en/docs/developers - - - id: feedjolt.boards.list - capability: product.feedback.boards.list - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/boards - name: "List boards in a workspace" - summary: "List Boards" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - test_request: - pathParams: {workspace_slug: "corvidly"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.posts.list - capability: product.feedback.posts.list - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/posts - name: "List posts in a workspace" - summary: "List Posts" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - queryParams: - page: {type: integer, required: false, example: 1} - page_size: {type: integer, required: false, note: "keep small", example: 5} - board_id: {type: string, required: false} - status_id: {type: string, required: false} - exclude_status_ids: {type: string, required: false} - tag_ids: {type: string, required: false} - q: {type: string, required: false, note: "minLength 2, maxLength 200"} - owner_admin_id: {type: string, required: false} - date_from: {type: string, required: false, note: "YYYY-MM-DD"} - date_to: {type: string, required: false, note: "YYYY-MM-DD"} - sort_by: {type: string, required: false, note: "default newest"} - test_request: - pathParams: {workspace_slug: "corvidly"} - queryParams: {page_size: 5} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.posts.search - capability: product.feedback.posts.search - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/posts/search - name: "Search posts in a workspace" - summary: "Search Posts" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - queryParams: - q: {type: string, required: true, example: "MCP", note: "minLength 2, maxLength 200"} - test_request: - pathParams: {workspace_slug: "corvidly"} - queryParams: {q: "MCP"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.posts.get - capability: product.feedback.posts.get - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/posts/{post_id} - name: "Get a post" - summary: "Get a post in a workspace" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - post_id: {type: string, required: true, example: "52428c9b-971a-4804-a96e-73ce3da4ab17"} - test_request: - pathParams: {workspace_slug: "corvidly", post_id: "52428c9b-971a-4804-a96e-73ce3da4ab17"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.roadmap.get - capability: product.feedback.roadmap.get - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/roadmap - name: "Get workspace roadmap" - summary: "Get Workspace Roadmap" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - test_request: - pathParams: {workspace_slug: "corvidly"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.changelog.list - capability: product.feedback.changelog.list - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/changelog - name: "List workspace changelog" - summary: "List Workspace Changelog" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - queryParams: - page: {type: integer, required: false, example: 1} - page_size: {type: integer, required: false, example: 5} - test_request: - pathParams: {workspace_slug: "corvidly"} - queryParams: {page_size: 5} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.statuses.list - capability: product.feedback.statuses.list - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/statuses - name: "List statuses" - summary: "List Statuses" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - test_request: - pathParams: {workspace_slug: "corvidly"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs - - - id: feedjolt.tags.list - capability: product.feedback.tags.list - platform: web - scope: own_account - method: GET - path: /workspaces/{workspace_slug}/tags - name: "List tags" - summary: "List Tags" - input: - pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - test_request: - pathParams: {workspace_slug: "corvidly"} - cost: - type: free - value: 0 - currency: USD - unit: call - source: docs - source_url: https://www.feedjolt.com/en - checked: "2026-09-02" - confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." - docs_url: https://api.feedjolt.com/docs From 69da9750a30f461a5a450427f97f95e1c14be0cd Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Wed, 2 Sep 2026 17:11:02 +0200 Subject: [PATCH 29/31] feedjolt: restore own_account GETs now that REST dual-auth is live --- src/treg/catalog/feedjolt.yaml | 416 +++++++++++++++++++++++++++++++++ 1 file changed, 416 insertions(+) diff --git a/src/treg/catalog/feedjolt.yaml b/src/treg/catalog/feedjolt.yaml index b711b2379..acbd54c21 100644 --- a/src/treg/catalog/feedjolt.yaml +++ b/src/treg/catalog/feedjolt.yaml @@ -25,3 +25,419 @@ proposed_capabilities: product.feedback.changelog.list: "List published changelog entries" product.feedback.statuses.list: "List feedback statuses in a workspace" product.feedback.tags.list: "List tags in a feedback workspace" + +endpoints: + - id: feedjolt.public.boards.list + capability: product.feedback.boards.list + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/boards + name: "List public boards" + summary: "List Public Boards" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read; no API key required." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.public.posts.list + capability: product.feedback.posts.list + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/boards/{board_slug}/posts + name: "List public posts on a board" + summary: "List Public Posts" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + board_slug: {type: string, required: true, example: "feedback"} + queryParams: + page: {type: integer, required: false, example: 1} + page_size: {type: integer, required: false, example: 3} + sort_by: {type: string, required: false} + q: {type: string, required: false} + status_id: {type: string, required: false} + tag_ids: {type: string, required: false} + test_request: + pathParams: {workspace_slug: "corvidly", board_slug: "feedback"} + queryParams: {page_size: 3} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read; no API key required." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.public.posts.get + capability: product.feedback.posts.get + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/posts/{post_id} + name: "Get a public post" + summary: "Get Public Post" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + post_id: {type: string, required: true, example: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + test_request: + pathParams: {workspace_slug: "corvidly", post_id: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read; no API key required." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.public.posts.search + capability: product.feedback.posts.search + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/search + name: "Search public posts" + summary: "Search Public Posts" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + q: {type: string, required: true, example: "MCP", note: "minLength 2, maxLength 200"} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {q: "MCP"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read. OpenAPI: q minLength 2, maxLength 200." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.public.roadmap.get + capability: product.feedback.roadmap.get + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/roadmap + name: "Get public roadmap" + summary: "Get Public Roadmap" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + tags: {type: string, required: false, note: "comma-separated tag ids"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.public.changelog.list + capability: product.feedback.changelog.list + platform: web + scope: any_account + method: GET + path: /public/{workspace_slug}/changelog + name: "List public changelog" + summary: "List Public Changelog" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + page: {type: integer, required: false, example: 1} + page_size: {type: integer, required: false, example: 5} + version_id: {type: string, required: false} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {page_size: 5} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Public portal read." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.account.workspaces.list + kind: account + capability: account.usage + platform: account + scope: own_account + method: GET + path: /workspaces + name: "List workspaces for this API key" + summary: "List Workspaces" + input: + note: "no parameters — the key rides in the Authorization Bearer header (X-API-Key also works). This is the registry probe." + test_request: {queryParams: {}} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Probe. Garbage Bearer/X-API-Key returns 401 {\"detail\":\"Invalid or revoked API key\"}. Cookie-missing (no auth) returns 401 {\"detail\":\"Not authenticated\"}. Valid-key 2xx pending maintainer credential." + docs_url: https://www.feedjolt.com/en/docs/developers + + - id: feedjolt.boards.list + capability: product.feedback.boards.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/boards + name: "List boards in a workspace" + summary: "List Boards" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.posts.list + capability: product.feedback.posts.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/posts + name: "List posts in a workspace" + summary: "List Posts" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + page: {type: integer, required: false, example: 1} + page_size: {type: integer, required: false, note: "keep small", example: 5} + board_id: {type: string, required: false} + status_id: {type: string, required: false} + exclude_status_ids: {type: string, required: false} + tag_ids: {type: string, required: false} + q: {type: string, required: false, note: "minLength 2, maxLength 200"} + owner_admin_id: {type: string, required: false} + date_from: {type: string, required: false, note: "YYYY-MM-DD"} + date_to: {type: string, required: false, note: "YYYY-MM-DD"} + sort_by: {type: string, required: false, note: "default newest"} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {page_size: 5} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.posts.search + capability: product.feedback.posts.search + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/posts/search + name: "Search posts in a workspace" + summary: "Search Posts" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + q: {type: string, required: true, example: "MCP", note: "minLength 2, maxLength 200"} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {q: "MCP"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.posts.get + capability: product.feedback.posts.get + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/posts/{post_id} + name: "Get a post" + summary: "Get a post in a workspace" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + post_id: {type: string, required: true, example: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + test_request: + pathParams: {workspace_slug: "corvidly", post_id: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.roadmap.get + capability: product.feedback.roadmap.get + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/roadmap + name: "Get workspace roadmap" + summary: "Get Workspace Roadmap" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.changelog.list + capability: product.feedback.changelog.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/changelog + name: "List workspace changelog" + summary: "List Workspace Changelog" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + queryParams: + page: {type: integer, required: false, example: 1} + page_size: {type: integer, required: false, example: 5} + test_request: + pathParams: {workspace_slug: "corvidly"} + queryParams: {page_size: 5} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.statuses.list + capability: product.feedback.statuses.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/statuses + name: "List statuses" + summary: "List Statuses" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs + + - id: feedjolt.tags.list + capability: product.feedback.tags.list + platform: web + scope: own_account + method: GET + path: /workspaces/{workspace_slug}/tags + name: "List tags" + summary: "List Tags" + input: + pathParams: + workspace_slug: {type: string, required: true, example: "corvidly"} + test_request: + pathParams: {workspace_slug: "corvidly"} + cost: + type: free + value: 0 + currency: USD + unit: call + source: docs + source_url: https://www.feedjolt.com/en + checked: "2026-09-02" + confidence: documented + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + docs_url: https://api.feedjolt.com/docs From 66709b5c35c4912528dd4e21b44b86590ad841fd Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Thu, 3 Sep 2026 08:37:50 +0200 Subject: [PATCH 30/31] feedjolt: stamp live 2xx, own_account examples to luodaint --- src/treg/catalog/feedjolt.yaml | 76 +++++++++++++++++----------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/src/treg/catalog/feedjolt.yaml b/src/treg/catalog/feedjolt.yaml index acbd54c21..41db3b896 100644 --- a/src/treg/catalog/feedjolt.yaml +++ b/src/treg/catalog/feedjolt.yaml @@ -2,7 +2,7 @@ provider: feedjolt source: docs: https://www.feedjolt.com/en/docs/developers openapi: https://api.feedjolt.com/openapi.json - curated: 2026-09-02 + curated: 2026-09-03 # BYOK listing. REST+MCP sit in Feedjolt's premium set (Startup $9/mo monthly, Scale $15/mo # annual). There is no per-call meter and no rate-card endpoint, so treg cannot serve these on # its own key yet. Authenticated routes are own_account workspace data. @@ -216,9 +216,9 @@ endpoints: unit: call source: docs source_url: https://www.feedjolt.com/en - checked: "2026-09-02" + checked: "2026-09-03" confidence: documented - note: "Probe. Garbage Bearer/X-API-Key returns 401 {\"detail\":\"Invalid or revoked API key\"}. Cookie-missing (no auth) returns 401 {\"detail\":\"Not authenticated\"}. Valid-key 2xx pending maintainer credential." + note: "Probe. Garbage Bearer/X-API-Key returns 401 {\"detail\":\"Invalid or revoked API key\"}. Cookie-missing (no auth) returns 401 {\"detail\":\"Not authenticated\"}. Matching-slug key returns 200 (observed 2026-09-03, workspace luodaint, Bearer and X-API-Key). Cross-workspace slug returns 403 {\"detail\":\"API key is not valid for this workspace\"}." docs_url: https://www.feedjolt.com/en/docs/developers - id: feedjolt.boards.list @@ -231,9 +231,9 @@ endpoints: summary: "List Boards" input: pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} + workspace_slug: {type: string, required: true, example: "luodaint"} test_request: - pathParams: {workspace_slug: "corvidly"} + pathParams: {workspace_slug: "luodaint"} cost: type: free value: 0 @@ -241,9 +241,9 @@ endpoints: unit: call source: docs source_url: https://www.feedjolt.com/en - checked: "2026-09-02" + checked: "2026-09-03" confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx on 2026-09-03 (slug must match the key). Cross-workspace slug returns 403." docs_url: https://api.feedjolt.com/docs - id: feedjolt.posts.list @@ -256,7 +256,7 @@ endpoints: summary: "List Posts" input: pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} + workspace_slug: {type: string, required: true, example: "luodaint"} queryParams: page: {type: integer, required: false, example: 1} page_size: {type: integer, required: false, note: "keep small", example: 5} @@ -270,7 +270,7 @@ endpoints: date_to: {type: string, required: false, note: "YYYY-MM-DD"} sort_by: {type: string, required: false, note: "default newest"} test_request: - pathParams: {workspace_slug: "corvidly"} + pathParams: {workspace_slug: "luodaint"} queryParams: {page_size: 5} cost: type: free @@ -279,9 +279,9 @@ endpoints: unit: call source: docs source_url: https://www.feedjolt.com/en - checked: "2026-09-02" + checked: "2026-09-03" confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx on 2026-09-03 (slug must match the key). Cross-workspace slug returns 403." docs_url: https://api.feedjolt.com/docs - id: feedjolt.posts.search @@ -294,12 +294,12 @@ endpoints: summary: "Search Posts" input: pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} + workspace_slug: {type: string, required: true, example: "luodaint"} queryParams: - q: {type: string, required: true, example: "MCP", note: "minLength 2, maxLength 200"} + q: {type: string, required: true, example: "ab", note: "minLength 2, maxLength 200"} test_request: - pathParams: {workspace_slug: "corvidly"} - queryParams: {q: "MCP"} + pathParams: {workspace_slug: "luodaint"} + queryParams: {q: "ab"} cost: type: free value: 0 @@ -307,9 +307,9 @@ endpoints: unit: call source: docs source_url: https://www.feedjolt.com/en - checked: "2026-09-02" + checked: "2026-09-03" confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx on 2026-09-03 (slug must match the key). Cross-workspace slug returns 403." docs_url: https://api.feedjolt.com/docs - id: feedjolt.posts.get @@ -322,10 +322,10 @@ endpoints: summary: "Get a post in a workspace" input: pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} - post_id: {type: string, required: true, example: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + workspace_slug: {type: string, required: true, example: "luodaint"} + post_id: {type: string, required: true, example: "b7e1779e-6c6d-4d07-b916-3dfd3ce5ad15"} test_request: - pathParams: {workspace_slug: "corvidly", post_id: "52428c9b-971a-4804-a96e-73ce3da4ab17"} + pathParams: {workspace_slug: "luodaint", post_id: "b7e1779e-6c6d-4d07-b916-3dfd3ce5ad15"} cost: type: free value: 0 @@ -333,9 +333,9 @@ endpoints: unit: call source: docs source_url: https://www.feedjolt.com/en - checked: "2026-09-02" + checked: "2026-09-03" confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx on 2026-09-03 (slug must match the key). Cross-workspace slug returns 403." docs_url: https://api.feedjolt.com/docs - id: feedjolt.roadmap.get @@ -348,9 +348,9 @@ endpoints: summary: "Get Workspace Roadmap" input: pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} + workspace_slug: {type: string, required: true, example: "luodaint"} test_request: - pathParams: {workspace_slug: "corvidly"} + pathParams: {workspace_slug: "luodaint"} cost: type: free value: 0 @@ -358,9 +358,9 @@ endpoints: unit: call source: docs source_url: https://www.feedjolt.com/en - checked: "2026-09-02" + checked: "2026-09-03" confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx on 2026-09-03 (slug must match the key). Cross-workspace slug returns 403." docs_url: https://api.feedjolt.com/docs - id: feedjolt.changelog.list @@ -373,12 +373,12 @@ endpoints: summary: "List Workspace Changelog" input: pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} + workspace_slug: {type: string, required: true, example: "luodaint"} queryParams: page: {type: integer, required: false, example: 1} page_size: {type: integer, required: false, example: 5} test_request: - pathParams: {workspace_slug: "corvidly"} + pathParams: {workspace_slug: "luodaint"} queryParams: {page_size: 5} cost: type: free @@ -387,9 +387,9 @@ endpoints: unit: call source: docs source_url: https://www.feedjolt.com/en - checked: "2026-09-02" + checked: "2026-09-03" confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx on 2026-09-03 (slug must match the key). Cross-workspace slug returns 403." docs_url: https://api.feedjolt.com/docs - id: feedjolt.statuses.list @@ -402,9 +402,9 @@ endpoints: summary: "List Statuses" input: pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} + workspace_slug: {type: string, required: true, example: "luodaint"} test_request: - pathParams: {workspace_slug: "corvidly"} + pathParams: {workspace_slug: "luodaint"} cost: type: free value: 0 @@ -412,9 +412,9 @@ endpoints: unit: call source: docs source_url: https://www.feedjolt.com/en - checked: "2026-09-02" + checked: "2026-09-03" confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx on 2026-09-03 (slug must match the key). Cross-workspace slug returns 403." docs_url: https://api.feedjolt.com/docs - id: feedjolt.tags.list @@ -427,9 +427,9 @@ endpoints: summary: "List Tags" input: pathParams: - workspace_slug: {type: string, required: true, example: "corvidly"} + workspace_slug: {type: string, required: true, example: "luodaint"} test_request: - pathParams: {workspace_slug: "corvidly"} + pathParams: {workspace_slug: "luodaint"} cost: type: free value: 0 @@ -437,7 +437,7 @@ endpoints: unit: call source: docs source_url: https://www.feedjolt.com/en - checked: "2026-09-02" + checked: "2026-09-03" confidence: documented - note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx pending maintainer credential." + note: "Included in plan. Auth: Authorization Bearer fjk_ (X-API-Key also works). Live 2xx on 2026-09-03 (slug must match the key). Cross-workspace slug returns 403." docs_url: https://api.feedjolt.com/docs From af3b24703dc9f2b752928bac0703c8532d516eac Mon Sep 17 00:00:00 2001 From: Marc Llopart Date: Thu, 3 Sep 2026 09:53:47 +0200 Subject: [PATCH 31/31] feedjolt: note OAuth 2.1 as main agent auth; REST listing still uses API keys --- src/treg/oauth_feedjolt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/treg/oauth_feedjolt.py b/src/treg/oauth_feedjolt.py index e6ea9c22b..14feed0ae 100644 --- a/src/treg/oauth_feedjolt.py +++ b/src/treg/oauth_feedjolt.py @@ -26,7 +26,8 @@ "Copy the key. It starts with fjk_.", ), setup_note=( - "REST calls are included in the plan, not billed per call. " + "Main agent auth is OAuth 2.1 on the MCP servers (DCR/PKCE; not this catalog). " + "This listing is REST: API keys (fjk_) also work. REST calls are included in the plan, not billed per call. " "GET /workspaces is the probe. Do not use GET /health — it is unauthenticated. " "A garbage Bearer/X-API-Key returns 401 {\"detail\":\"Invalid or revoked API key\"}." ),