From 8a4140d450cd4f8604d38209997dae7b4072e0c4 Mon Sep 17 00:00:00 2001 From: Steffen Zellmer <151627820+Steffen025@users.noreply.github.com> Date: Mon, 16 Mar 2026 01:18:18 +0100 Subject: [PATCH 1/5] feat: reorganize skills into Utilities/ namespace (Teil 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move 130 skill files into Utilities/ subdirectory structure: - Aphorisms → Utilities/Aphorisms - Browser → Utilities/Browser - Cloudflare → Utilities/Cloudflare - CreateCLI → Utilities/CreateCLI - CreateSkill → Utilities/CreateSkill - Documents (Docx, Pdf, Pptx, Xlsx) → Utilities/Documents - Evals → Utilities/Evals - Prompting → Utilities/Prompting - SECUpdates → Utilities/SECUpdates - System → Utilities/System - VoiceServer → Utilities/VoiceServer Part of PAI v3.0 skill reorganization (PR-07 of 12) --- .../Utilities/Aphorisms/Database/aphorisms.md | 214 +++ .opencode/skills/Utilities/Aphorisms/SKILL.md | 383 +++++ .../Aphorisms/Workflows/AddAphorism.md | 539 +++++++ .../Aphorisms/Workflows/FindAphorism.md | 490 +++++++ .../Aphorisms/Workflows/ResearchThinker.md | 624 ++++++++ .../Aphorisms/Workflows/SearchAphorisms.md | 720 ++++++++++ .opencode/skills/Utilities/Browser/README.md | 230 +++ .opencode/skills/Utilities/Browser/SKILL.md | 268 ++++ .../skills/Utilities/Browser/Tools/Browse.ts | 694 +++++++++ .../Utilities/Browser/Tools/BrowserSession.ts | 445 ++++++ .../Utilities/Browser/Workflows/Extract.md | 125 ++ .../Utilities/Browser/Workflows/Interact.md | 115 ++ .../Utilities/Browser/Workflows/Screenshot.md | 71 + .../Utilities/Browser/Workflows/Update.md | 86 ++ .../Utilities/Browser/Workflows/VerifyPage.md | 94 ++ .opencode/skills/Utilities/Browser/bun.lock | 30 + .../Browser/examples/comprehensive-test.ts | 417 ++++++ .../Utilities/Browser/examples/screenshot.ts | 71 + .../Utilities/Browser/examples/verify-page.ts | 91 ++ .opencode/skills/Utilities/Browser/index.ts | 907 ++++++++++++ .../skills/Utilities/Browser/package.json | 16 + .../skills/Utilities/Browser/tsconfig.json | 16 + .../skills/Utilities/Cloudflare/SKILL.md | 105 ++ .../Utilities/Cloudflare/Workflows/Create.md | 76 + .../Cloudflare/Workflows/Troubleshoot.md | 555 +++++++ .../CreateCLI/FrameworkComparison.md | 477 ++++++ .../skills/Utilities/CreateCLI/Patterns.md | 487 +++++++ .opencode/skills/Utilities/CreateCLI/SKILL.md | 363 +++++ .../Utilities/CreateCLI/TypescriptPatterns.md | 786 ++++++++++ .../CreateCLI/Workflows/AddCommand.md | 161 +++ .../CreateCLI/Workflows/CreateCli.md | 813 +++++++++++ .../CreateCLI/Workflows/UpgradeTier.md | 177 +++ .../skills/Utilities/CreateSkill/SKILL.md | 261 ++++ .../Workflows/CanonicalizeSkill.md | 311 ++++ .../CreateSkill/Workflows/CreateSkill.md | 226 +++ .../CreateSkill/Workflows/UpdateSkill.md | 136 ++ .../CreateSkill/Workflows/ValidateSkill.md | 225 +++ .../skills/Utilities/Delegation/SKILL.md | 189 +++ .opencode/skills/Utilities/Documents/SKILL.md | 298 ++++ .../Workflows/ProcessLargePdfGemini3.md | 673 +++++++++ .opencode/skills/Utilities/Docx/LICENSE.txt | 30 + .../Utilities/Docx/Ooxml/Scripts/pack.py | 159 ++ .../Utilities/Docx/Ooxml/Scripts/unpack.py | 29 + .../Utilities/Docx/Ooxml/Scripts/validate.py | 69 + .opencode/skills/Utilities/Docx/SKILL.md | 260 ++++ .../skills/Utilities/Docx/Scripts/__init__.py | 1 + .../skills/Utilities/Docx/Scripts/document.py | 1276 +++++++++++++++++ .../Utilities/Docx/Scripts/utilities.py | 374 +++++ .opencode/skills/Utilities/Docx/docx-js.md | 350 +++++ .opencode/skills/Utilities/Docx/ooxml.md | 610 ++++++++ .../skills/Utilities/Evals/BestPractices.md | 48 + .../skills/Utilities/Evals/CLIReference.md | 89 ++ .../Utilities/Evals/Data/DomainPatterns.yaml | 157 ++ .../skills/Utilities/Evals/Graders/Base.ts | 122 ++ .../Evals/Graders/CodeBased/BinaryTests.ts | 77 + .../Evals/Graders/CodeBased/RegexMatch.ts | 64 + .../Evals/Graders/CodeBased/StateCheck.ts | 154 ++ .../Evals/Graders/CodeBased/StaticAnalysis.ts | 88 ++ .../Evals/Graders/CodeBased/StringMatch.ts | 54 + .../Graders/CodeBased/ToolCallVerification.ts | 112 ++ .../Evals/Graders/CodeBased/index.ts | 19 + .../Evals/Graders/ModelBased/LLMRubric.ts | 184 +++ .../ModelBased/NaturalLanguageAssert.ts | 122 ++ .../Graders/ModelBased/PairwiseComparison.ts | 160 +++ .../Evals/Graders/ModelBased/index.ts | 13 + .../skills/Utilities/Evals/Graders/index.ts | 16 + .opencode/skills/Utilities/Evals/PROJECT.md | 876 +++++++++++ .opencode/skills/Utilities/Evals/SKILL.md | 229 +++ .../skills/Utilities/Evals/ScienceMapping.md | 54 + .../skills/Utilities/Evals/ScorerTypes.md | 62 + .../Suites/Regression/core-behaviors.yaml | 18 + .../Utilities/Evals/TemplateIntegration.md | 72 + .../Utilities/Evals/Tools/AlgorithmBridge.ts | 237 +++ .../Utilities/Evals/Tools/FailureToTask.ts | 373 +++++ .../Utilities/Evals/Tools/SuiteManager.ts | 385 +++++ .../Evals/Tools/TranscriptCapture.ts | 215 +++ .../Utilities/Evals/Tools/TrialRunner.ts | 273 ++++ .../skills/Utilities/Evals/Types/index.ts | 387 +++++ .../Regression/task_file_targeting_basic.yaml | 53 + .../task_no_hallucinated_paths.yaml | 48 + .../task_tool_sequence_read_before_edit.yaml | 50 + .../task_verification_before_done.yaml | 50 + .../Evals/Workflows/CompareModels.md | 317 ++++ .../Evals/Workflows/ComparePrompts.md | 338 +++++ .../Utilities/Evals/Workflows/CreateJudge.md | 197 +++ .../Evals/Workflows/CreateUseCase.md | 288 ++++ .../Utilities/Evals/Workflows/RunEval.md | 106 ++ .../Utilities/Evals/Workflows/ViewResults.md | 373 +++++ .../skills/Utilities/PAIUpgrade/SKILL.md | 363 +++++ .../Utilities/PAIUpgrade/Tools/Anthropic.ts | 879 ++++++++++++ .../PAIUpgrade/Workflows/CheckForUpgrades.md | 175 +++ .../PAIUpgrade/Workflows/FindSources.md | 229 +++ .../Workflows/ReleaseNotesDeepDive.md | 197 +++ .../PAIUpgrade/Workflows/ResearchUpgrade.md | 219 +++ .../skills/Utilities/PAIUpgrade/sources.json | 184 +++ .../PAIUpgrade/youtube-channels.json | 3 + .../skills/Utilities/Parser/EntitySystem.md | 105 ++ .../skills/Utilities/Parser/Lib/parser.ts | 305 ++++ .../skills/Utilities/Parser/Lib/validators.ts | 299 ++++ .../Parser/Prompts/entity-extraction.md | 204 +++ .../Utilities/Parser/Prompts/link-analysis.md | 289 ++++ .../Utilities/Parser/Prompts/summarization.md | 211 +++ .../Parser/Prompts/topic-classification.md | 258 ++++ .opencode/skills/Utilities/Parser/README.md | 231 +++ .opencode/skills/Utilities/Parser/SKILL.md | 124 ++ .../Parser/Schema/content-schema.json | 518 +++++++ .../skills/Utilities/Parser/Schema/schema.ts | 223 +++ .../Parser/Tests/fixtures/example-output.json | 200 +++ .../Parser/Utils/collision-detection.ts | 323 +++++ .../skills/Utilities/Parser/Web/README.md | 150 ++ .../skills/Utilities/Parser/Web/debug.html | 34 + .../skills/Utilities/Parser/Web/index.html | 66 + .../skills/Utilities/Parser/Web/parser.js | 526 +++++++ .../Utilities/Parser/Web/simple-test.html | 62 + .../skills/Utilities/Parser/Web/styles.css | 498 +++++++ .../Workflows/BatchEntityExtractionGemini3.md | 757 ++++++++++ .../Parser/Workflows/CollisionDetection.md | 291 ++++ .../Parser/Workflows/DetectContentType.md | 339 +++++ .../Parser/Workflows/ExtractArticle.md | 474 ++++++ .../Workflows/ExtractBrowserExtension.md | 335 +++++ .../Parser/Workflows/ExtractNewsletter.md | 320 +++++ .../Utilities/Parser/Workflows/ExtractPdf.md | 346 +++++ .../Parser/Workflows/ExtractTwitter.md | 429 ++++++ .../Parser/Workflows/ExtractYoutube.md | 386 +++++ .../Parser/Workflows/ParseContent.md | 413 ++++++ .../skills/Utilities/Parser/entity-index.json | 8 + .opencode/skills/Utilities/Pdf/LICENSE.txt | 30 + .opencode/skills/Utilities/Pdf/SKILL.md | 440 ++++++ .../Pdf/Scripts/check_bounding_boxes.py | 70 + .../Pdf/Scripts/check_bounding_boxes_test.py | 226 +++ 130 files changed, 34342 insertions(+) create mode 100755 .opencode/skills/Utilities/Aphorisms/Database/aphorisms.md create mode 100755 .opencode/skills/Utilities/Aphorisms/SKILL.md create mode 100755 .opencode/skills/Utilities/Aphorisms/Workflows/AddAphorism.md create mode 100755 .opencode/skills/Utilities/Aphorisms/Workflows/FindAphorism.md create mode 100755 .opencode/skills/Utilities/Aphorisms/Workflows/ResearchThinker.md create mode 100755 .opencode/skills/Utilities/Aphorisms/Workflows/SearchAphorisms.md create mode 100755 .opencode/skills/Utilities/Browser/README.md create mode 100755 .opencode/skills/Utilities/Browser/SKILL.md create mode 100755 .opencode/skills/Utilities/Browser/Tools/Browse.ts create mode 100755 .opencode/skills/Utilities/Browser/Tools/BrowserSession.ts create mode 100755 .opencode/skills/Utilities/Browser/Workflows/Extract.md create mode 100755 .opencode/skills/Utilities/Browser/Workflows/Interact.md create mode 100755 .opencode/skills/Utilities/Browser/Workflows/Screenshot.md create mode 100755 .opencode/skills/Utilities/Browser/Workflows/Update.md create mode 100755 .opencode/skills/Utilities/Browser/Workflows/VerifyPage.md create mode 100755 .opencode/skills/Utilities/Browser/bun.lock create mode 100755 .opencode/skills/Utilities/Browser/examples/comprehensive-test.ts create mode 100755 .opencode/skills/Utilities/Browser/examples/screenshot.ts create mode 100755 .opencode/skills/Utilities/Browser/examples/verify-page.ts create mode 100755 .opencode/skills/Utilities/Browser/index.ts create mode 100755 .opencode/skills/Utilities/Browser/package.json create mode 100755 .opencode/skills/Utilities/Browser/tsconfig.json create mode 100644 .opencode/skills/Utilities/Cloudflare/SKILL.md create mode 100644 .opencode/skills/Utilities/Cloudflare/Workflows/Create.md create mode 100644 .opencode/skills/Utilities/Cloudflare/Workflows/Troubleshoot.md create mode 100755 .opencode/skills/Utilities/CreateCLI/FrameworkComparison.md create mode 100755 .opencode/skills/Utilities/CreateCLI/Patterns.md create mode 100755 .opencode/skills/Utilities/CreateCLI/SKILL.md create mode 100755 .opencode/skills/Utilities/CreateCLI/TypescriptPatterns.md create mode 100755 .opencode/skills/Utilities/CreateCLI/Workflows/AddCommand.md create mode 100755 .opencode/skills/Utilities/CreateCLI/Workflows/CreateCli.md create mode 100755 .opencode/skills/Utilities/CreateCLI/Workflows/UpgradeTier.md create mode 100755 .opencode/skills/Utilities/CreateSkill/SKILL.md create mode 100755 .opencode/skills/Utilities/CreateSkill/Workflows/CanonicalizeSkill.md create mode 100755 .opencode/skills/Utilities/CreateSkill/Workflows/CreateSkill.md create mode 100755 .opencode/skills/Utilities/CreateSkill/Workflows/UpdateSkill.md create mode 100755 .opencode/skills/Utilities/CreateSkill/Workflows/ValidateSkill.md create mode 100644 .opencode/skills/Utilities/Delegation/SKILL.md create mode 100755 .opencode/skills/Utilities/Documents/SKILL.md create mode 100755 .opencode/skills/Utilities/Documents/Workflows/ProcessLargePdfGemini3.md create mode 100755 .opencode/skills/Utilities/Docx/LICENSE.txt create mode 100755 .opencode/skills/Utilities/Docx/Ooxml/Scripts/pack.py create mode 100755 .opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py create mode 100755 .opencode/skills/Utilities/Docx/Ooxml/Scripts/validate.py create mode 100755 .opencode/skills/Utilities/Docx/SKILL.md create mode 100755 .opencode/skills/Utilities/Docx/Scripts/__init__.py create mode 100755 .opencode/skills/Utilities/Docx/Scripts/document.py create mode 100755 .opencode/skills/Utilities/Docx/Scripts/utilities.py create mode 100755 .opencode/skills/Utilities/Docx/docx-js.md create mode 100755 .opencode/skills/Utilities/Docx/ooxml.md create mode 100755 .opencode/skills/Utilities/Evals/BestPractices.md create mode 100755 .opencode/skills/Utilities/Evals/CLIReference.md create mode 100755 .opencode/skills/Utilities/Evals/Data/DomainPatterns.yaml create mode 100755 .opencode/skills/Utilities/Evals/Graders/Base.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/CodeBased/BinaryTests.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/CodeBased/RegexMatch.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/CodeBased/StateCheck.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/CodeBased/StaticAnalysis.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/CodeBased/StringMatch.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/CodeBased/ToolCallVerification.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/CodeBased/index.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/ModelBased/LLMRubric.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/ModelBased/NaturalLanguageAssert.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/ModelBased/PairwiseComparison.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/ModelBased/index.ts create mode 100755 .opencode/skills/Utilities/Evals/Graders/index.ts create mode 100755 .opencode/skills/Utilities/Evals/PROJECT.md create mode 100755 .opencode/skills/Utilities/Evals/SKILL.md create mode 100755 .opencode/skills/Utilities/Evals/ScienceMapping.md create mode 100755 .opencode/skills/Utilities/Evals/ScorerTypes.md create mode 100755 .opencode/skills/Utilities/Evals/Suites/Regression/core-behaviors.yaml create mode 100755 .opencode/skills/Utilities/Evals/TemplateIntegration.md create mode 100755 .opencode/skills/Utilities/Evals/Tools/AlgorithmBridge.ts create mode 100755 .opencode/skills/Utilities/Evals/Tools/FailureToTask.ts create mode 100755 .opencode/skills/Utilities/Evals/Tools/SuiteManager.ts create mode 100755 .opencode/skills/Utilities/Evals/Tools/TranscriptCapture.ts create mode 100755 .opencode/skills/Utilities/Evals/Tools/TrialRunner.ts create mode 100755 .opencode/skills/Utilities/Evals/Types/index.ts create mode 100755 .opencode/skills/Utilities/Evals/UseCases/Regression/task_file_targeting_basic.yaml create mode 100755 .opencode/skills/Utilities/Evals/UseCases/Regression/task_no_hallucinated_paths.yaml create mode 100755 .opencode/skills/Utilities/Evals/UseCases/Regression/task_tool_sequence_read_before_edit.yaml create mode 100755 .opencode/skills/Utilities/Evals/UseCases/Regression/task_verification_before_done.yaml create mode 100755 .opencode/skills/Utilities/Evals/Workflows/CompareModels.md create mode 100755 .opencode/skills/Utilities/Evals/Workflows/ComparePrompts.md create mode 100755 .opencode/skills/Utilities/Evals/Workflows/CreateJudge.md create mode 100755 .opencode/skills/Utilities/Evals/Workflows/CreateUseCase.md create mode 100755 .opencode/skills/Utilities/Evals/Workflows/RunEval.md create mode 100755 .opencode/skills/Utilities/Evals/Workflows/ViewResults.md create mode 100755 .opencode/skills/Utilities/PAIUpgrade/SKILL.md create mode 100755 .opencode/skills/Utilities/PAIUpgrade/Tools/Anthropic.ts create mode 100755 .opencode/skills/Utilities/PAIUpgrade/Workflows/CheckForUpgrades.md create mode 100755 .opencode/skills/Utilities/PAIUpgrade/Workflows/FindSources.md create mode 100755 .opencode/skills/Utilities/PAIUpgrade/Workflows/ReleaseNotesDeepDive.md create mode 100755 .opencode/skills/Utilities/PAIUpgrade/Workflows/ResearchUpgrade.md create mode 100755 .opencode/skills/Utilities/PAIUpgrade/sources.json create mode 100755 .opencode/skills/Utilities/PAIUpgrade/youtube-channels.json create mode 100755 .opencode/skills/Utilities/Parser/EntitySystem.md create mode 100755 .opencode/skills/Utilities/Parser/Lib/parser.ts create mode 100755 .opencode/skills/Utilities/Parser/Lib/validators.ts create mode 100755 .opencode/skills/Utilities/Parser/Prompts/entity-extraction.md create mode 100755 .opencode/skills/Utilities/Parser/Prompts/link-analysis.md create mode 100755 .opencode/skills/Utilities/Parser/Prompts/summarization.md create mode 100755 .opencode/skills/Utilities/Parser/Prompts/topic-classification.md create mode 100755 .opencode/skills/Utilities/Parser/README.md create mode 100755 .opencode/skills/Utilities/Parser/SKILL.md create mode 100755 .opencode/skills/Utilities/Parser/Schema/content-schema.json create mode 100755 .opencode/skills/Utilities/Parser/Schema/schema.ts create mode 100755 .opencode/skills/Utilities/Parser/Tests/fixtures/example-output.json create mode 100755 .opencode/skills/Utilities/Parser/Utils/collision-detection.ts create mode 100755 .opencode/skills/Utilities/Parser/Web/README.md create mode 100755 .opencode/skills/Utilities/Parser/Web/debug.html create mode 100755 .opencode/skills/Utilities/Parser/Web/index.html create mode 100755 .opencode/skills/Utilities/Parser/Web/parser.js create mode 100755 .opencode/skills/Utilities/Parser/Web/simple-test.html create mode 100755 .opencode/skills/Utilities/Parser/Web/styles.css create mode 100755 .opencode/skills/Utilities/Parser/Workflows/BatchEntityExtractionGemini3.md create mode 100755 .opencode/skills/Utilities/Parser/Workflows/CollisionDetection.md create mode 100755 .opencode/skills/Utilities/Parser/Workflows/DetectContentType.md create mode 100755 .opencode/skills/Utilities/Parser/Workflows/ExtractArticle.md create mode 100644 .opencode/skills/Utilities/Parser/Workflows/ExtractBrowserExtension.md create mode 100755 .opencode/skills/Utilities/Parser/Workflows/ExtractNewsletter.md create mode 100755 .opencode/skills/Utilities/Parser/Workflows/ExtractPdf.md create mode 100755 .opencode/skills/Utilities/Parser/Workflows/ExtractTwitter.md create mode 100755 .opencode/skills/Utilities/Parser/Workflows/ExtractYoutube.md create mode 100755 .opencode/skills/Utilities/Parser/Workflows/ParseContent.md create mode 100755 .opencode/skills/Utilities/Parser/entity-index.json create mode 100755 .opencode/skills/Utilities/Pdf/LICENSE.txt create mode 100755 .opencode/skills/Utilities/Pdf/SKILL.md create mode 100755 .opencode/skills/Utilities/Pdf/Scripts/check_bounding_boxes.py create mode 100755 .opencode/skills/Utilities/Pdf/Scripts/check_bounding_boxes_test.py diff --git a/.opencode/skills/Utilities/Aphorisms/Database/aphorisms.md b/.opencode/skills/Utilities/Aphorisms/Database/aphorisms.md new file mode 100755 index 00000000..2cb6d991 --- /dev/null +++ b/.opencode/skills/Utilities/Aphorisms/Database/aphorisms.md @@ -0,0 +1,214 @@ +# Aphorism Database + +Complete collection of curated aphorisms organized by theme and author. + +## Initial Collection (Rahil Arora) + +### Hard Work & Excellence + +**"You don't have to love the hard work. You just have to crave the result so intensely that the hard work becomes irrelevant."** +- Author: Tim Grover +- Theme: Work ethic, Excellence, Results-oriented +- Context: From "Relentless" - Champions focus on outcomes, not effort + +**"I hated every minute of training, but I said, 'Don't quit. Suffer now and live the rest of your life as a champion."** +- Author: Muhammad Ali +- Theme: Discipline, Sacrifice, Excellence +- Context: Boxing legend on the price of greatness + +**"The pain you feel today will be the strength you feel tomorrow."** +- Author: Unknown +- Theme: Growth, Resilience, Delayed gratification +- Context: Physical and mental transformation through struggle + +**"You never know how strong you are, until being strong is your only choice."** +- Author: Bob Marley +- Theme: Resilience, Necessity, Strength discovery +- Context: Adversity reveals hidden capabilities + +--- + +### Fear & Action + +**"Fear is excitement without breath."** +- Author: Robert Heller +- Theme: Fear reframing, Physiology, Mindset +- Context: Fear and excitement are physiologically identical - breathe to transform + +--- + +### Passion & Enthusiasm + +**"I began to realize how important it was to be an enthusiast in life. He taught me that if you are interested in something, no matter what it is, go at it at full speed ahead. Embrace it with both arms, hug it, love it and above all become passionate about it. Lukewarm is no good."** +- Author: Roald Dahl +- Theme: Passion, Enthusiasm, Full commitment +- Context: From "My Uncle Oswald" - No half measures in life + +--- + +### Competition & Progress + +**"Run your own race. Who cares what others are doing? The only question that matters is 'Am I progressing?'"** +- Author: Robin S. Sharma +- Theme: Self-competition, Progress, Focus +- Context: The Monk Who Sold His Ferrari - Internal benchmarks matter + +--- + +### Curiosity & Intelligence + +**"Be curious. Read widely. Try new things. What people call intelligence just boils down to curiosity."** +- Author: Aaron Swartz +- Theme: Curiosity, Learning, Intelligence +- Context: Internet activist defining intelligence as active curiosity + +--- + +### Investment & Self-Development + +**"Investing in yourself is the best investment you will ever make. it will not only improve your life, it will improve the lives of all those around you."** +- Author: Robin S. Sharma +- Theme: Self-investment, Impact, Growth +- Context: Leadership wisdom - personal growth creates ripple effects + +--- + +### Present Moment & Enjoyment + +**"Learn to enjoy every minute of your life. Be happy now. Don't wait for something outside of yourself to make you happy in the future. Think how really precious is the time you have to spend, whether it's at work or with your family. Every minute should be enjoyed and savored."** +- Author: Earl Nightingale +- Theme: Present moment, Happiness, Appreciation +- Context: Personal development pioneer on mindfulness and gratitude + +--- + +### Learning & Living + +**"Live as if you were to die tomorrow. Learn as if you were to live forever."** +- Author: Mahatma Gandhi +- Theme: Urgency, Learning, Balance +- Context: Combining immediate action with long-term knowledge pursuit + +--- + +### Mindset & Capability + +**"Your 'I CAN' is more important than your IQ."** +- Author: Robin S. Sharma +- Theme: Mindset, Self-belief, Growth mindset +- Context: Attitude determines altitude more than raw intelligence + +--- + +### Lifelong Education + +**"There is no end to education. It is not that you read a book, pass an examination, and finish with education. The whole of life, from the moment you are born to the moment you die, is a process of learning."** +- Author: Jiddu Krishnamurti +- Theme: Lifelong learning, Education philosophy, Continuous growth +- Context: Philosopher on education as life itself, not credentials + +--- + +### Learning & Thinking + +**"He who learns but does not think, is lost! He who thinks but does not learn is in great danger."** +- Author: Confucius +- Theme: Balance, Critical thinking, Applied knowledge +- Context: Ancient wisdom on integrating knowledge with reflection + +--- + +### Adversity & Challenge + +**"The world ain't all sunshine and rainbows. It is a very mean and nasty place It will beat you to your knees and keep you there permanently if you let it. You, me or nobody is going to hit as hard as life. But it ain't about how hard you're hit, it is about how hard you can get hit and keep moving forward, how much can you take and keep moving forward. That's how winning is done!"** +- Author: Rocky Balboa (2006) +- Theme: Resilience, Persistence, Life's challenges +- Context: Iconic movie speech on perseverance and grit + +--- + +### Stoicism & Control + +**"You have power over your mind - not outside events. Realize this, and you will find strength."** +- Author: Marcus Aurelius +- Theme: Stoicism, Internal locus of control, Mental strength +- Context: Meditations - Roman emperor on the dichotomy of control + +--- + +### Risk & Failure + +**"If you try, you risk failure. If you don't, you ensure it."** +- Author: Unknown +- Theme: Risk-taking, Failure, Inaction +- Context: The guarantee of failure through inaction vs possibility through action + +--- + +## Thinkers Aligned with TELOS Philosophy + +### Christopher Hitchens +*Quotes to be added from research* + +### David Deutsch +*Quotes to be added from research* + +### Sam Harris +*Quotes to be added from research* + +### Baruch Spinoza +*Quotes to be added from research* + +### Richard Feynman +*Quotes to be added from research* + +--- + +## Theme Index + +**Work Ethic & Excellence:** +- Tim Grover, Muhammad Ali + +**Resilience & Strength:** +- Unknown (pain/strength), Bob Marley, Rocky Balboa + +**Fear & Mindset:** +- Robert Heller, Robin S. Sharma (I CAN) + +**Passion & Enthusiasm:** +- Roald Dahl + +**Progress & Competition:** +- Robin S. Sharma (run your own race) + +**Curiosity & Intelligence:** +- Aaron Swartz + +**Self-Investment:** +- Robin S. Sharma (investing) + +**Present Moment:** +- Earl Nightingale + +**Learning & Education:** +- Gandhi, Jiddu Krishnamurti, Confucius + +**Stoicism & Control:** +- Marcus Aurelius + +**Risk & Action:** +- Unknown (try vs don't) + +**Adversity:** +- Rocky Balboa + +--- + +## Newsletter Usage History + +*Track which aphorisms have been used in which newsletters to avoid repetition* + +--- + +## Last Updated +2025-11-20 diff --git a/.opencode/skills/Utilities/Aphorisms/SKILL.md b/.opencode/skills/Utilities/Aphorisms/SKILL.md new file mode 100755 index 00000000..0eeda2da --- /dev/null +++ b/.opencode/skills/Utilities/Aphorisms/SKILL.md @@ -0,0 +1,383 @@ +--- +name: Aphorisms +description: Aphorism management. USE WHEN aphorism, quote, saying. SkillSearch('aphorisms') for docs. +--- + +## Customization + +**Before executing, check for user customizations at:** +`~/.opencode/skills/PAI/USER/SKILLCUSTOMIZATIONS/Aphorisms/` + +If this directory exists, load and apply any PREFERENCES.md, configurations, or resources found there. These override default behavior. If the directory does not exist, proceed with skill defaults. + + +## 🚨 MANDATORY: Voice Notification (REQUIRED BEFORE ANY ACTION) + +**You MUST send this notification BEFORE doing anything else when this skill is invoked.** + +1. **Send voice notification**: + ```bash + curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the WORKFLOWNAME workflow in the Aphorisms skill to ACTION"}' \ + > /dev/null 2>&1 & + ``` + +2. **Output text notification**: + ``` + Running the **WorkflowName** workflow in the **Aphorisms** skill to ACTION... + ``` + +**This is not optional. Execute this curl command immediately upon skill invocation.** + +## Workflow Routing + +**When executing a workflow, output this notification directly:** + +``` +Running the **WorkflowName** workflow in the **Aphorisms** skill to ACTION... +``` + +**When user requests finding perfect aphorism for newsletter content:** +Examples: "find aphorism for this newsletter", "find quote for this content", "what aphorism fits this", "suggest quote for newsletter", "match aphorism to this article", "perfect quote for this", "aphorism recommendation" +→ **READ:** ~/.opencode/skills/aphorisms/Workflows/Find-aphorism.md +→ **EXECUTE:** Analyze content themes and recommend matching aphorism from database + +**When user requests adding new aphorism to database:** +Examples: "add this quote", "add aphorism", "save this quote", "add to aphorism database", "new aphorism", "store this quote", "include this in collection" +→ **READ:** ~/.opencode/skills/aphorisms/Workflows/Add-aphorism.md +→ **EXECUTE:** Add new aphorism with proper metadata and theme tagging + +**When user requests researching specific thinker's quotes:** +Examples: "research Hitchens quotes", "find Feynman aphorisms", "what did Spinoza say about", "get quotes from Sam Harris", "research David Deutsch wisdom", "thinker quotes on [topic]" +→ **READ:** ~/.opencode/skills/aphorisms/Workflows/Research-thinker.md +→ **EXECUTE:** Research thinker's relevant quotes and add to database + +**When user requests searching aphorisms by theme or keyword:** +Examples: "search aphorisms about resilience", "find quotes on learning", "aphorisms about stoicism", "quotes matching [keyword]", "show me quotes about [theme]", "what aphorisms do we have on" +→ **READ:** ~/.opencode/skills/aphorisms/Workflows/Search-aphorisms.md +→ **EXECUTE:** Search database by theme, keyword, or author + +--- + +## When to Activate This Skill + +### Direct Aphorism Requests +- "find aphorism", "find a quote", "find quote for X" +- "search aphorisms", "search quotes", "look up quote" +- "what aphorism", "which quote", "perfect quote for" +- "suggest aphorism", "recommend quote", "match quote to" +- "aphorism for newsletter", "quote for blog post", "quote for article" + +### Database Management +- "add aphorism", "add quote", "save this quote" +- "new aphorism", "include quote", "store this" +- "update aphorism database", "manage quotes" + +### Research & Discovery +- "research [thinker] quotes", "find [author] aphorisms" +- "what did [philosopher] say about", "quotes from [thinker]" +- "Hitchens quotes", "Feynman wisdom", "Spinoza aphorisms" +- "Sam Harris on [topic]", "David Deutsch quotes" + +### Theme-Based Search +- "aphorisms about [theme]", "quotes on [topic]" +- "show quotes about resilience", "wisdom on learning" +- "stoic quotes", "quotes matching [keyword]" + +### Newsletter Workflow Integration +- User working on newsletter and needs aphorism +- Mentions "newsletter" + "quote" or "aphorism" +- Content analysis for quote matching +- Avoiding previously used quotes + +### Use Case Indicators +- Need wisdom quote to open/close newsletter +- Want thematically relevant aphorism +- Building quote collection +- Researching philosopher's ideas +- Managing aphorism library + +--- + +## Core Capabilities + +### 1. Intelligent Quote Matching +Analyze newsletter or article content to find the perfect thematic aphorism: +- Extract key themes from content +- Match themes to aphorism database +- Consider tone and style alignment +- Avoid recently used quotes +- Provide multiple options with rationale + +### 2. Comprehensive Database +Curated collection organized by: +- **Author** - Thinkers aligned with TELOS philosophy +- **Theme** - Categories like resilience, learning, stoicism, risk, progress +- **Context** - Background on quote origin and meaning +- **Usage History** - Track which quotes used in which newsletters + +### 3. Thinker Research +Deep research on key philosophers: +- **Christopher Hitchens** - Rationality, skepticism, intellectual honesty +- **David Deutsch** - Knowledge creation, optimism, explanations +- **Sam Harris** - Rationality, meditation, free will, morality +- **Baruch Spinoza** - Ethics, reason, freedom, nature +- **Richard Feynman** - Curiosity, scientific thinking, doubt, clarity + +### 4. Theme-Based Organization +Aphorisms categorized by themes matching user content: +- **Work Ethic & Excellence** - Craft, mastery, high standards +- **Resilience & Strength** - Adversity, persistence, growth +- **Learning & Education** - Curiosity, continuous improvement +- **Stoicism & Control** - Internal locus, acceptance, discipline +- **Risk & Action** - Courage, failure, experimentation +- **Wisdom & Truth** - Rationality, evidence, honest inquiry + +--- + +## Database Structure + +**Location:** `~/.opencode/skills/aphorisms/Database/aphorisms.md` + +**Current Collections:** +1. **Initial Collection (Rahil Arora)** - 15 curated quotes covering core themes +2. **Thinkers Aligned with TELOS** - Sections for Hitchens, Deutsch, Harris, Spinoza, Feynman (to be populated) +3. **Theme Index** - Quick reference by category +4. **Newsletter Usage History** - Tracking to avoid repetition + +**Metadata Per Aphorism:** +- Full quote text +- Author attribution +- Theme tags +- Context and background +- Source reference (when available) + +--- + +## Available Workflows + +### Quote Discovery & Matching + +**find-aphorism.md** - Intelligent newsletter content analysis +- Analyze content themes and tone +- Search database for thematic matches +- Consider usage history +- Provide top 3-5 recommendations with rationale +- Include quote, author, and why it fits + +### Database Management + +**add-aphorism.md** - Structured quote addition +- Accept quote text and author +- Extract or assign themes +- Add context and background +- Update theme index +- Validate uniqueness + +### Research Operations + +**research-thinker.md** - Deep thinker research +- Research specific philosopher's relevant quotes +- Focus on TELOS-aligned themes +- Add quotes to appropriate database section +- Include context and sources +- Update theme index + +### Search & Discovery + +**search-aphorisms.md** - Theme and keyword search +- Search by theme, keyword, or author +- Return matching aphorisms +- Sort by relevance or usage +- Provide context for each result + +--- + +## Integration Points + +### Newsletter Content Skill +- Automatic aphorism suggestions when creating newsletter +- Theme analysis from newsletter content +- Usage tracking for variety + +### Research Skill +- Deep thinker research capabilities +- Web research for quote verification +- Source attribution and context + +### Writing Skill +- Blog post quote recommendations +- Story explanation enhancement +- Content opening/closing quotes + +--- + +## Key Thinkers & Philosophy Alignment + +### Why These Thinkers? + +All five thinkers align with TELOS themes of **wisdom, rationality, truth-seeking, and human flourishing:** + +**Christopher Hitchens** +- Intellectual honesty and skepticism +- Question everything, follow evidence +- "What can be asserted without evidence can be dismissed without evidence" + +**David Deutsch** +- Optimistic epistemology - problems are solvable +- Knowledge creation through criticism +- Emphasis on explanations, not just predictions + +**Sam Harris** +- Scientific rationality applied to ethics +- Importance of reason and evidence +- Mindfulness and self-awareness + +**Baruch Spinoza** +- Ethics based on reason +- Freedom through understanding +- Reality acceptance and wisdom + +**Richard Feynman** +- Curiosity-driven learning +- Doubt as a tool for knowledge +- Clarity of thought and explanation +- Scientific honesty + +### Research Priority + +1. **Immediate**: Analyze previous newsletters for aphorism patterns +2. **Phase 1**: Research Hitchens and Feynman (most quotable, clear style) +3. **Phase 2**: Research Harris and Deutsch (contemporary, relevant) +4. **Phase 3**: Research Spinoza (historical, philosophical depth) + +--- + +## Usage Examples + +### Example 1: Finding Aphorism for Newsletter + +**User:** "I'm writing a newsletter about overcoming setbacks in AI research. Find me a good aphorism." + +**Skill Response:** +1. Analyze themes: resilience, adversity, persistence, progress +2. Search database for matching themes +3. Recommend top 3 options: + - Rocky Balboa quote (direct, powerful on getting hit and moving forward) + - Bob Marley quote (strength through necessity) + - Marcus Aurelius quote (stoic control focus) +4. Provide rationale for each + +### Example 2: Adding New Quote + +**User:** "Add this quote: 'The cure for boredom is curiosity. There is no cure for curiosity.' - Dorothy Parker" + +**Skill Response:** +1. Parse quote and author +2. Identify themes: curiosity, learning, passion +3. Add to database with context +4. Update theme index +5. Confirm addition + +### Example 3: Researching Thinker + +**User:** "Research David Deutsch quotes about knowledge and optimism" + +**Skill Response:** +1. Research Deutsch's works (The Beginning of Infinity, The Fabric of Reality) +2. Extract relevant quotes on knowledge creation and optimism +3. Add to database with source attribution +4. Organize by theme +5. Report findings + +### Example 4: Theme Search + +**User:** "Show me all aphorisms about learning and education" + +**Skill Response:** +1. Search database for learning/education theme +2. Return matching quotes: + - Gandhi (live/learn) + - Krishnamurti (lifelong learning) + - Confucius (learning + thinking) + - Aaron Swartz (curiosity) +3. Provide context for each + +--- + +## Best Practices + +### Quote Selection for Newsletter +1. **Match tone** - Ensure quote tone aligns with newsletter content +2. **Thematic relevance** - Direct connection to main themes +3. **Avoid repetition** - Check usage history +4. **Provide variety** - Rotate between authors and themes +5. **Context matters** - Consider whether reader needs background + +### Database Maintenance +1. **Verify accuracy** - Check quote text and attribution +2. **Add context** - Include source and background when possible +3. **Theme consistently** - Use established theme categories +4. **Track usage** - Update history to avoid overuse +5. **Quality over quantity** - Curate, don't just collect + +### Thinker Research +1. **Primary sources** - Prefer direct quotes from books/speeches +2. **Context critical** - Include enough background for understanding +3. **Avoid misattribution** - Verify quote authenticity +4. **TELOS alignment** - Focus on wisdom, rationality, truth-seeking +5. **Practical wisdom** - Quotes should be actionable or profound + +--- + +## Future Enhancements + +### Planned Features +1. **Automatic theme detection** - ML-based content analysis +2. **Quote recommendation engine** - Collaborative filtering based on past selections +3. **Integration with previous newsletters** - Analyze historical aphorism usage patterns +4. **Expanded thinker research** - Add more philosophers aligned with TELOS +5. **Mood/tone matching** - Match quote emotional tone to content +6. **Quote formatting** - Auto-format for newsletter style + +### Long-term Vision +- Comprehensive wisdom library covering all content needs +- Predictive recommendations based on newsletter draft +- Historical analysis of most impactful quotes +- Community contributions (vetted) +- Integration with other writing workflows + +--- + +## Quick Reference + +**Most Used Commands:** +- "Find aphorism for this newsletter" → Analyze content and recommend +- "Add this quote" → Add to database with metadata +- "Research [thinker] quotes" → Deep research and database population +- "Search aphorisms about [theme]" → Theme-based search + +**Database Location:** +`~/.opencode/skills/aphorisms/Database/aphorisms.md` + +**Current Collection Size:** +- 15 initial quotes (Rahil Arora collection) +- 5 thinker sections (to be populated) +- 12+ theme categories + +**Key Thinkers:** +Hitchens, Deutsch, Harris, Spinoza, Feynman + +--- + +## Related Skills + +**newsletter-content** - Newsletter creation and content suggestions +**research** - Web research and content analysis +**writing** - Blog post and content creation +**personal** - User's philosophy and values context + +--- + +Last Updated: 2025-11-20 diff --git a/.opencode/skills/Utilities/Aphorisms/Workflows/AddAphorism.md b/.opencode/skills/Utilities/Aphorisms/Workflows/AddAphorism.md new file mode 100755 index 00000000..63aac542 --- /dev/null +++ b/.opencode/skills/Utilities/Aphorisms/Workflows/AddAphorism.md @@ -0,0 +1,539 @@ +# Add Aphorism to Database + +**Purpose:** Add new aphorism to the database with proper metadata, theme tagging, and organization. + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the AddAphorism workflow in the Aphorisms skill to add quotes"}' \ + > /dev/null 2>&1 & +``` + +Running **AddAphorism** in **Aphorisms**... + +--- + +**When to Use:** +- User provides a quote to add to collection +- User says "add this quote", "save this aphorism", "add to database" +- Found great quote during research that should be preserved +- After research-thinker.md discovers quotes worth adding + +**Prerequisites:** +- Aphorism database exists at `~/.opencode/skills/aphorisms/Database/aphorisms.md` +- Quote text and author provided (or discoverable through research) +- Database is Read first to check for duplicates + +--- + +## Workflow Steps + +### Step 1: Parse Input + +**Required Information:** +- **Quote Text** - Full quote in exact wording +- **Author** - Person who said/wrote it + +**Optional Information (will research if not provided):** +- **Themes** - Categories this quote fits +- **Context** - Background, source, when it was said +- **Source** - Book, speech, interview, etc. + +**Example Input Formats:** + +```markdown +User: "Add this quote: 'The cure for boredom is curiosity. There is no cure for curiosity.' - Dorothy Parker" + +User: "Save this to the database: +Quote: Walk away from anything or anyone who takes away from your joy. Life is too short to put up with fools. +Author: Unknown" + +User: "Add Feynman's quote about doubt being essential to science" +(Will need to research specific wording) +``` + +--- + +### Step 2: Verify Quote Accuracy + +**If exact quote text provided:** +- Use WebSearch to verify wording and attribution +- Check for common misattributions +- Confirm author if "Unknown" or uncertain + +**If paraphrased or remembered:** +- Research to find exact wording +- Verify correct source +- Get full quote if user only provided partial + +**Verification Steps:** +```bash +# Search for exact quote +WebSearch("\"[quote text]\" [author name] quote") + +# Verify attribution +WebSearch("[author name] quotes + [key words from quote]") + +# Check common misattributions +WebSearch("misattributed quotes [author name]") +``` + +**Output:** +- ✅ Verified exact text +- ✅ Confirmed correct attribution +- ✅ Source identified (if available) + +--- + +### Step 3: Check for Duplicates + +**Read database:** +```bash +Read ~/.opencode/skills/aphorisms/Database/aphorisms.md +``` + +**Check for:** +1. **Exact duplicate** - Same quote already exists +2. **Similar quote** - Same author, similar message +3. **Paraphrase** - Different wording, same meaning + +**If duplicate found:** +- Notify user: "This quote is already in the database in [section]" +- Ask: "Would you like to update context or themes?" +- OR: "This is similar to [existing quote]. Add anyway?" + +**If unique:** +- Proceed to Step 4 + +--- + +### Step 4: Analyze Themes + +**Identify Primary Themes:** + +Use the established theme categories: +- **Work Ethic & Excellence** - Craft, mastery, high standards, results +- **Resilience & Strength** - Adversity, persistence, growth through challenge +- **Fear & Mindset** - Overcoming fear, reframing, mental approach +- **Passion & Enthusiasm** - Full commitment, love for craft, intensity +- **Competition & Progress** - Self-improvement, personal growth, benchmarks +- **Curiosity & Intelligence** - Learning, questioning, intellectual drive +- **Investment & Self-Development** - Personal growth, skill building +- **Present Moment & Enjoyment** - Mindfulness, appreciation, being here now +- **Learning & Education** - Knowledge acquisition, continuous improvement +- **Stoicism & Control** - Internal locus, acceptance, discipline +- **Risk & Action** - Courage, failure acceptance, experimentation +- **Adversity** - Challenges, setbacks, overcoming difficulty + +**Assign 1-3 primary themes:** +- Must be specific and accurate +- Err on side of fewer, more accurate themes vs many vague ones +- Consider what newsletter topics this would fit + +**Example Analysis:** + +Quote: "The cure for boredom is curiosity. There is no cure for curiosity." +**Themes:** +1. Curiosity & Intelligence (PRIMARY) +2. Passion & Enthusiasm (SECONDARY) + +Quote: "Walk away from anything or anyone who takes away from your joy." +**Themes:** +1. Present Moment & Enjoyment (PRIMARY) +2. Stoicism & Control (SECONDARY - choosing what affects you) + +--- + +### Step 5: Add Context + +**Provide background for the quote:** + +**Context should include:** +1. **Source** - Where did this quote come from? + - Book title, speech, interview, letter, etc. + - Year if known + +2. **Background** - What was happening when this was said? + - Circumstances + - Original intended meaning + - Why it's significant + +3. **Relevance** - Why does this matter for the audience? + - How it applies to modern life + - Connection to TELOS philosophy + - Practical wisdom it provides + +**If context not immediately known:** +```bash +WebSearch("[author name] '[quote snippet]' context source") +WebSearch("[author name] biography + [time period/work]") +``` + +**Example Context:** + +Quote: "You have power over your mind - not outside events." +**Context:** +- Source: Meditations by Marcus Aurelius (Book 12, written ~170 AD) +- Background: Written as personal reflections while on military campaign +- Relevance: Stoic philosophy's dichotomy of control - foundational to mental strength and resilience + +--- + +### Step 6: Format for Database + +**Standard Format:** + +```markdown +**"[Full quote text]"** +- Author: [Author Name] +- Theme: [Theme 1], [Theme 2], [Theme 3] +- Context: [Source and background] +``` + +**Example:** + +```markdown +**"The cure for boredom is curiosity. There is no cure for curiosity."** +- Author: Dorothy Parker +- Theme: Curiosity & Intelligence, Passion & Enthusiasm +- Context: American poet and wit (1893-1967). Captures the endless nature of intellectual curiosity - the more you learn, the more you want to learn. +``` + +--- + +### Step 7: Determine Placement + +**Where in database does this go?** + +**Option 1: Add to Theme Section** +- If matching an existing theme collection +- Place alphabetically by author within theme + +**Option 2: Add to Thinker Section** +- If quote is from one of the five key thinkers (Hitchens, Deutsch, Harris, Spinoza, Feynman) +- Add to appropriate thinker's section + +**Option 3: Add to Initial Collection** +- If general quote not from key thinker +- Add chronologically to main collection + +**Option 4: Create New Section** +- If this starts a new category or author collection +- Propose new section to user first + +--- + +### Step 8: Update Theme Index + +**Theme Index Section:** + +Add quote reference to appropriate theme(s) in Theme Index section: + +```markdown +## Theme Index + +**Curiosity & Intelligence:** +- Aaron Swartz, Dorothy Parker +``` + +**Update format:** +- Add author to theme list (alphabetically) +- Keep index concise (just author names) +- Multiple themes = author appears multiple times + +--- + +### Step 9: Write to Database + +**Use Edit tool to add quote:** + +```bash +# Find appropriate section +Read ~/.opencode/skills/aphorisms/Database/aphorisms.md + +# Add to correct location +Edit( + file_path=~/.opencode/skills/aphorisms/Database/aphorisms.md, + old_string="[section where it should be inserted]", + new_string="[section with new quote added]" +) +``` + +**Update multiple sections:** +1. Add formatted quote to main collection +2. Update theme index with author +3. If usage tracking enabled, initialize empty entry + +--- + +### Step 10: Confirm Addition + +**Summary for User:** + +```markdown +✅ **Aphorism Added Successfully** + +**Quote:** "[quote text]" +**Author:** [Author Name] +**Themes:** [Theme 1], [Theme 2] + +**Added to:** +- Main Collection: [Section name] +- Theme Index: Updated [N] theme(s) + +**Database Stats:** +- Total aphorisms: [N] +- Authors: [N] +- Themes covered: [N] + +**Ready to use in:** Newsletter recommendations, theme searches +``` + +--- + +## Advanced Features + +### Batch Addition + +If user provides multiple quotes: + +```markdown +User: "Add these three quotes from Feynman: +1. [Quote 1] +2. [Quote 2] +3. [Quote 3]" +``` + +**Process:** +1. Parse all quotes +2. Verify each (may need research) +3. Analyze themes for all +4. Add context to each +5. Add all to database in single Edit +6. Update theme index once +7. Provide batch summary + +### Research-Driven Addition + +If quote text not provided: + +```markdown +User: "Add Spinoza's quote about freedom through understanding" +``` + +**Process:** +1. Research Spinoza quotes on freedom/understanding +2. Find exact quote text +3. Present options to user if multiple matches +4. User selects preferred version +5. Proceed with normal addition workflow + +### Update Existing Quote + +If quote exists but needs better context: + +```markdown +User: "Update the Marcus Aurelius quote with more context" +``` + +**Process:** +1. Find quote in database +2. Research additional context +3. Edit existing entry +4. Preserve themes and attribution +5. Confirm update + +--- + +## Quality Checks + +Before finalizing addition: + +- [ ] Quote text verified as accurate +- [ ] Attribution verified as correct +- [ ] No exact duplicates in database +- [ ] 1-3 appropriate themes assigned +- [ ] Context provided (source, background, relevance) +- [ ] Formatted correctly for database +- [ ] Placed in appropriate section +- [ ] Theme index updated +- [ ] Addition confirmed to user + +--- + +## Edge Cases + +### What if Author is Uncertain? + +**Use "Unknown" or "Disputed":** +```markdown +**"[Quote text]"** +- Author: Unknown (often attributed to [Name 1], [Name 2]) +- Theme: [Themes] +- Context: This quote is commonly attributed to [Name] but cannot be verified. [Background on why quote is valuable despite uncertain origin] +``` + +### What if Quote Has Multiple Versions? + +**Choose canonical version or add note:** +```markdown +**"[Primary version]"** +- Author: [Author Name] +- Theme: [Themes] +- Context: [Background]. Note: This quote appears in multiple variations; this is the most commonly cited version from [Source]. +``` + +### What if Quote is Misattributed? + +**Correct attribution:** +```markdown +User: "Add Einstein quote: 'Everyone is a genius...'" +Research: Actually from Unknown author, falsely attributed to Einstein + +Response: "This quote is commonly misattributed to Einstein but has no verified source. Would you like to: +1. Add it as 'Unknown' with note about misattribution +2. Skip adding this quote +3. Research similar authentic Einstein quotes" +``` + +### What if Themes Don't Fit Existing Categories? + +**Propose new category:** +```markdown +"This quote about [topic] doesn't fit our existing themes well. Options: +1. Add to closest theme: [Theme X] +2. Create new theme category: [Proposed Theme] +3. Add to general collection without specific theme + +Which would you prefer?" +``` + +--- + +## Integration with Other Workflows + +### After Addition + +**Available for:** +- **find-aphorism.md** - Immediately searchable for newsletter matching +- **search-aphorisms.md** - Discoverable via theme/keyword search + +### Bulk Import + +If adding many quotes from research: +1. Use research-thinker.md to gather quotes +2. Then batch add via this workflow +3. Maintain quality checks for each + +### Usage Tracking + +When quote is used in newsletter: +```markdown +## Newsletter Usage History + +**"[Quote text]" - [Author]** +- Used in: 2025-11-20 Newsletter "Overcoming Setbacks" +- Placement: Opening quote +``` + +--- + +## Success Criteria + +**Addition succeeds when:** +- Quote accurately captured with correct attribution +- Appropriate themes assigned +- Context provides value for future selection +- Database organization maintained +- Theme index updated +- No duplicates created +- User receives confirmation + +**Quality indicators:** +- Quote is immediately useful for find-aphorism workflow +- Future search will surface this quote for relevant themes +- Context helps understand when/how to use it +- Attribution is verifiable + +--- + +## Example Execution + +### User Input +"Add this quote: 'The important thing is not to stop questioning. Curiosity has its own reason for existing.' - Einstein" + +### Step 1: Parse +- Quote: "The important thing is not to stop questioning. Curiosity has its own reason for existing." +- Author: Albert Einstein + +### Step 2: Verify +```bash +WebSearch("\"The important thing is not to stop questioning\" Einstein") +``` +**Result:** Verified - from 1955 interview + +### Step 3: Check Duplicates +Read database - NOT FOUND, unique quote + +### Step 4: Analyze Themes +**Primary Themes:** +1. Curiosity & Intelligence (PRIMARY - directly about questioning/curiosity) +2. Learning & Education (SECONDARY - implies lifelong learning) + +### Step 5: Add Context +**Context:** From Einstein's 1955 interview "Old Man's Advice to Youth." Emphasizes intrinsic value of curiosity beyond practical applications. + +### Step 6: Format +```markdown +**"The important thing is not to stop questioning. Curiosity has its own reason for existing."** +- Author: Albert Einstein +- Theme: Curiosity & Intelligence, Learning & Education +- Context: From 1955 interview "Old Man's Advice to Youth." Emphasizes intrinsic value of curiosity beyond practical applications. +``` + +### Step 7: Determine Placement +- Add to Initial Collection (Einstein not one of five key thinkers) +- Add chronologically + +### Step 8: Update Theme Index +```markdown +**Curiosity & Intelligence:** +- Aaron Swartz, Dorothy Parker, Albert Einstein +``` + +### Step 9: Write to Database +Use Edit tool to insert in appropriate section + +### Step 10: Confirm +```markdown +✅ **Aphorism Added Successfully** + +**Quote:** "The important thing is not to stop questioning. Curiosity has its own reason for existing." +**Author:** Albert Einstein +**Themes:** Curiosity & Intelligence, Learning & Education + +**Added to:** +- Main Collection: Initial Collection (Curiosity & Intelligence section) +- Theme Index: Updated 2 themes + +**Database Stats:** +- Total aphorisms: 16 +- Authors: 14 +- Themes covered: 12 + +**Ready to use in:** Newsletter recommendations, theme searches +``` + +--- + +## Related Workflows + +- **find-aphorism.md** - Use newly added quotes in recommendations +- **search-aphorisms.md** - Search will now find this quote +- **research-thinker.md** - Often feeds into this workflow + +--- + +**Last Updated:** 2025-11-20 diff --git a/.opencode/skills/Utilities/Aphorisms/Workflows/FindAphorism.md b/.opencode/skills/Utilities/Aphorisms/Workflows/FindAphorism.md new file mode 100755 index 00000000..e5acf070 --- /dev/null +++ b/.opencode/skills/Utilities/Aphorisms/Workflows/FindAphorism.md @@ -0,0 +1,490 @@ +# Find Aphorism for Newsletter Content + +**Purpose:** Analyze newsletter or article content to recommend the perfect thematically-aligned aphorism from the database. + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the FindAphorism workflow in the Aphorisms skill to find quotes"}' \ + > /dev/null 2>&1 & +``` + +Running **FindAphorism** in **Aphorisms**... + +--- + +**When to Use:** +- User provides newsletter draft or URL and requests aphorism +- User describes newsletter theme and wants quote recommendation +- User says "find aphorism for this", "what quote fits this", "suggest quote" +- Working on newsletter and needs opening/closing wisdom quote + +**Prerequisites:** +- Aphorism database exists at `~/.opencode/skills/aphorisms/Database/aphorisms.md` +- Newsletter content or URL provided by user +- Clear understanding of newsletter theme (if not provided, extract from content) + +--- + +## Workflow Steps + +### Step 1: Get Newsletter Content + +**If URL provided:** +```bash +# Use WebFetch to get content +WebFetch(url, "Extract main article content, title, and key themes") +``` + +**If content pasted:** +- Receive full text directly from user + +**If theme described:** +- Work with theme description only (e.g., "newsletter about overcoming setbacks") + +**Expected Outcome:** +- Full newsletter text OR clear theme description + +--- + +### Step 2: Analyze Content Themes + +**Extract Primary Themes:** + +Use deep thinking for deep thematic analysis. Identify: + +1. **Core Topic** - What is the newsletter fundamentally about? + - Examples: AI safety, personal productivity, security vulnerabilities, market analysis + +2. **Emotional Tone** - What's the mood/feeling? + - Examples: Optimistic, cautionary, reflective, urgent, inspirational + +3. **Key Messages** - What are the 2-3 main takeaways? + - Examples: "Persistence matters more than talent", "Question assumptions", "Focus on fundamentals" + +4. **Philosophical Alignment** - Which TELOS themes are present? + - Wisdom & Truth-seeking + - Rationality & Evidence + - Human flourishing & Progress + - Resilience & Growth + - Learning & Curiosity + - Stoicism & Control + - Risk & Action + - Excellence & Mastery + +5. **Audience Context** - Who is this for? + - Unsupervised Learning newsletter subscribers (technical, curious, rationalist) + - Blog readers (varied technical background) + - Social media audience (quick insights) + +**Analysis Output:** +```markdown +### Content Analysis +- **Core Topic**: [Topic] +- **Emotional Tone**: [Tone] +- **Key Messages**: + 1. [Message 1] + 2. [Message 2] + 3. [Message 3] +- **TELOS Themes**: [Theme 1], [Theme 2], [Theme 3] +- **Audience**: [Context] +``` + +--- + +### Step 3: Read Aphorism Database + +**Load database:** +```bash +Read ~/.opencode/skills/aphorisms/Database/aphorisms.md +``` + +**Review relevant sections:** +1. Check theme index for matching categories +2. Review aphorisms in matching theme categories +3. Note usage history to avoid recently used quotes + +**Expected Outcome:** +- Full database context loaded +- Theme-relevant sections identified + +--- + +### Step 4: Match Aphorisms to Themes + +**Matching Criteria:** + +For each potential aphorism, score on: + +1. **Thematic Relevance** (0-10) + - Does the quote directly address the newsletter's core themes? + - Strong connection = 8-10 + - Tangential connection = 4-7 + - Weak connection = 0-3 + +2. **Tonal Alignment** (0-10) + - Does the quote's mood match the newsletter's tone? + - Perfect match = 8-10 + - Compatible = 5-7 + - Mismatched = 0-4 + +3. **Message Reinforcement** (0-10) + - Does the quote strengthen the newsletter's key messages? + - Strongly reinforces = 8-10 + - Somewhat supports = 4-7 + - Neutral or contradictory = 0-3 + +4. **Philosophical Alignment** (0-10) + - Does the quote embody TELOS philosophy? + - Deep alignment = 8-10 + - Some alignment = 5-7 + - Misaligned = 0-4 + +5. **Freshness** (0-10) + - Has this quote been used recently? + - Not used recently = 10 + - Used 6+ months ago = 7-9 + - Used within 6 months = 3-6 + - Used within 3 months = 0-2 + +**Total Score:** Sum of all criteria (max 50) + +**Select Top 3-5 Candidates:** +- All candidates should score 35+ (70%) +- Prefer variety in authors +- Balance well-known vs lesser-known quotes + +--- + +### Step 5: Generate Recommendations + +**Format for Each Recommendation:** + +```markdown +### Recommendation [N]: [Quote Summary] + +**Quote:** +"[Full quote text]" + +**Author:** [Author Name] + +**Why This Fits:** +- **Thematic Relevance**: [Specific connection to newsletter themes] +- **Tonal Alignment**: [How mood/style matches] +- **Message Support**: [Which key message it reinforces] +- **TELOS Alignment**: [Which philosophy themes it embodies] + +**Placement Suggestion:** +[Opening quote / Closing quote / Section divider] - [Reasoning] + +**Context (if needed):** +[Brief background on quote or author if readers might need it] + +**Score:** [X/50] +``` + +**Provide 3-5 Recommendations:** +- **#1: Best Overall Match** - Highest total score +- **#2: Alternative Strong Match** - Second-best or different tone +- **#3: Safe Classic** - Well-known, universally resonant +- **#4 (optional): Unexpected Choice** - High relevance but lesser-known author/quote +- **#5 (optional): Philosophical Depth** - For readers seeking deeper wisdom + +--- + +### Step 6: Present to User + +**Summary Format:** + +```markdown +## Aphorism Recommendations for "[Newsletter Title/Theme]" + +### Content Analysis Summary +[Brief 2-3 sentence summary of themes and tone] + +### Top Recommendations + +[Include all recommendations from Step 5] + +### Quick Decision Guide +- **For maximum impact**: Recommendation #1 +- **For variety** (if #1 author used recently): Recommendation #2 +- **For broad appeal**: Recommendation #3 +- **For surprise/delight**: Recommendation #4 +- **For philosophical depth**: Recommendation #5 + +### Usage Tracking +Remember to update usage history in database after selection. +``` + +--- + +## Advanced Techniques + +### Multi-Newsletter Analysis + +If user has multiple newsletter drafts: +1. Analyze all content simultaneously +2. Find thematic through-lines +3. Recommend aphorisms that span themes +4. Ensure variety across newsletters + +### Seasonal/Temporal Context + +Consider: +- Time of year (e.g., New Year = fresh starts, growth quotes) +- Current events (avoid tone-deaf selections) +- Recent newsletter themes (ensure variety) + +### Author Diversity + +Track author usage over time: +- Avoid overusing same author +- Rotate between classical and contemporary +- Balance well-known and obscure thinkers + +--- + +## Common Patterns + +### Pattern 1: Resilience/Adversity Newsletter +**Common Themes:** Setbacks, challenges, persistence, growth through difficulty +**Go-To Quotes:** Rocky Balboa, Marcus Aurelius, Muhammad Ali, Bob Marley +**Tone:** Inspirational, empowering, realistic about difficulty + +### Pattern 2: Learning/Curiosity Newsletter +**Common Themes:** Knowledge acquisition, continuous improvement, intellectual honesty +**Go-To Quotes:** Feynman, Aaron Swartz, Confucius, Krishnamurti +**Tone:** Encouraging, thoughtful, emphasizing growth mindset + +### Pattern 3: Action/Risk Newsletter +**Common Themes:** Taking chances, overcoming fear, experimenting, moving forward +**Go-To Quotes:** "If you try, you risk failure", Robert Heller (fear = excitement), Tim Grover +**Tone:** Bold, action-oriented, courage-focused + +### Pattern 4: Excellence/Mastery Newsletter +**Common Themes:** High standards, craft, dedication, results over effort +**Go-To Quotes:** Tim Grover, Muhammad Ali, Robin Sharma (investment) +**Tone:** Demanding, uncompromising, focused on outcomes + +### Pattern 5: Stoicism/Control Newsletter +**Common Themes:** Focus on what you can control, acceptance, internal strength +**Go-To Quotes:** Marcus Aurelius, Spinoza (when added) +**Tone:** Calm, philosophical, centered + +### Pattern 6: Progress/Competition Newsletter +**Common Themes:** Self-improvement, internal benchmarks, personal growth +**Go-To Quotes:** Robin Sharma (run your own race), Gandhi (live/learn) +**Tone:** Encouraging, progress-focused, non-comparative + +--- + +## Quality Checks + +Before finalizing recommendations: + +- [ ] All quotes verified for accuracy (correct text and attribution) +- [ ] Thematic relevance is clear and specific +- [ ] Tonal alignment makes sense (no jarring mismatches) +- [ ] TELOS philosophy alignment is genuine +- [ ] Usage history checked (not recently used) +- [ ] Context provided if quote needs background +- [ ] Placement suggestion is appropriate +- [ ] Author diversity maintained +- [ ] At least 3 recommendations provided +- [ ] Recommendations are ranked/explained clearly + +--- + +## Edge Cases + +### What if No Perfect Match? + +**Option 1: Expand search to related themes** +- Look for adjacent themes (e.g., resilience → strength → adversity) +- Consider complementary rather than identical matches + +**Option 2: Research new quotes** +- Use research-thinker.md workflow to find relevant quotes from key thinkers +- Add new quotes to database and recommend + +**Option 3: Use philosophical principles** +- Match to higher-level TELOS themes (wisdom, rationality, flourishing) +- Recommend quotes that align philosophically even if not directly on-topic + +### What if User Rejects All Recommendations? + +**Clarify preferences:** +- Ask: "What didn't resonate about these options?" +- Understand: Tone issue? Author issue? Message misalignment? +- Adjust: Provide new recommendations based on feedback + +**Research on demand:** +- Offer to research specific thinker or theme +- Expand beyond current database +- Find exactly what user envisions + +### What if Content is Multi-Themed? + +**Two approaches:** + +1. **Pick dominant theme** - Recommend quote matching primary theme +2. **Find unifying quote** - Recommend quote that speaks to multiple themes + +**Example:** +Newsletter about "AI safety through careful engineering" +- Themes: Safety, caution, excellence, responsibility +- Unifying quote: Feynman on scientific honesty and rigor (when added) +- OR Marcus Aurelius on control and wisdom + +--- + +## Integration with Other Workflows + +### After Recommendation is Selected + +**Update database:** +1. Use add-aphorism.md to update usage history +2. Note: Newsletter date, newsletter theme, placement +3. Maintain tracking for future variety + +### If Quote Not in Database + +**Add new quote:** +1. User provides or requests research for specific quote +2. Use add-aphorism.md to add with full metadata +3. Then recommend for current newsletter + +### If Need Expanded Collection + +**Research thinkers:** +1. Use research-thinker.md to populate thinker sections +2. Focus on themes matching newsletter needs +3. Build database for future recommendations + +--- + +## Success Criteria + +**Workflow succeeds when:** +- User receives 3-5 high-quality, relevant recommendations +- Each recommendation has clear rationale +- Thematic relevance is obvious +- User can confidently select a quote +- Selected quote enhances newsletter impact + +**Quality indicators:** +- User says "perfect!" or "exactly what I needed" +- Minimal back-and-forth required +- Quote scores 35+ on matching criteria +- TELOS alignment is clear +- Recommendation saves user research time + +--- + +## Example Execution + +### User Input +"I'm writing a newsletter about how major security breaches happen from ignoring basic security hygiene. Find me a good aphorism." + +### Step 1: Content Analysis +```markdown +### Content Analysis +- **Core Topic**: Security fundamentals, basics matter +- **Emotional Tone**: Cautionary, frustrated, emphasizing fundamentals +- **Key Messages**: + 1. Basics matter more than advanced techniques + 2. Ignoring fundamentals leads to failure + 3. Excellence requires discipline on basics +- **TELOS Themes**: Excellence & Mastery, Discipline, Learning (back to basics) +- **Audience**: Security professionals, technical readers +``` + +### Step 2: Database Search +Read database, identify matches in: +- "Excellence & Mastery" theme +- "Learning & Education" theme (fundamentals) +- Quotes emphasizing discipline and basics + +### Step 3: Top Matches +1. Tim Grover (hard work = results) - 38/50 +2. Muhammad Ali (suffer now = champion) - 37/50 +3. Confucius (learning + thinking) - 35/50 + +### Step 4: Recommendations + +```markdown +### Recommendation #1: Focus on What Matters (Tim Grover) + +**Quote:** +"You don't have to love the hard work. You just have to crave the result so intensely that the hard work becomes irrelevant." + +**Why This Fits:** +- **Thematic Relevance**: Security hygiene IS hard work - tedious, unglamorous, but critical +- **Tonal Alignment**: Uncompromising, results-focused matches newsletter's emphasis on fundamentals +- **Message Support**: Reinforces "basics matter" - the "hard work" IS the basics +- **TELOS Alignment**: Excellence through discipline, results > effort + +**Placement Suggestion:** +Opening quote - Sets tone that security fundamentals are non-negotiable hard work + +**Score:** 38/50 + +### Recommendation #2: The Price of Excellence (Muhammad Ali) + +**Quote:** +"I hated every minute of training, but I said, 'Don't quit. Suffer now and live the rest of your life as a champion.'" + +**Why This Fits:** +- **Thematic Relevance**: Training = security hygiene practice - boring but prevents breaches +- **Tonal Alignment**: Direct, honest about difficulty, emphasizes long-term payoff +- **Message Support**: "Suffer now" (do the boring security work) = "live as champion" (avoid breaches) +- **TELOS Alignment**: Delayed gratification, discipline, mastery + +**Placement Suggestion:** +Closing quote - Motivates readers to commit to security fundamentals despite tedium + +**Score:** 37/50 + +### Recommendation #3: Balance Theory and Practice (Confucius) + +**Quote:** +"He who learns but does not think, is lost! He who thinks but does not learn is in great danger." + +**Why This Fits:** +- **Thematic Relevance**: Security requires BOTH knowledge and applied practice (hygiene) +- **Tonal Alignment**: Warning tone matches cautionary newsletter mood +- **Message Support**: "Does not learn" (ignores basics) = "great danger" (breaches happen) +- **TELOS Alignment**: Integration of knowledge and action, wisdom through practice + +**Placement Suggestion:** +Section divider - Connects learning about security to actually implementing hygiene + +**Score:** 35/50 +``` + +### Step 5: Quick Decision +```markdown +### Quick Decision Guide +- **For maximum impact on security professionals**: #1 (Tim Grover) +- **For motivational close**: #2 (Muhammad Ali) +- **For philosophical depth**: #3 (Confucius) +``` + +**User selects:** Tim Grover quote + +**Update tracking:** Note usage in database + +--- + +## Related Workflows + +- **add-aphorism.md** - Update usage history after selection +- **research-thinker.md** - Expand database if no good matches +- **search-aphorisms.md** - Explore database by theme before analysis + +--- + +**Last Updated:** 2025-11-20 diff --git a/.opencode/skills/Utilities/Aphorisms/Workflows/ResearchThinker.md b/.opencode/skills/Utilities/Aphorisms/Workflows/ResearchThinker.md new file mode 100755 index 00000000..0249d8c8 --- /dev/null +++ b/.opencode/skills/Utilities/Aphorisms/Workflows/ResearchThinker.md @@ -0,0 +1,624 @@ +# Research Thinker Quotes + +**Purpose:** Deep research on specific philosopher/thinker to discover relevant aphorisms aligned with TELOS philosophy, then add to database. + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the ResearchThinker workflow in the Aphorisms skill to research quotes"}' \ + > /dev/null 2>&1 & +``` + +Running **ResearchThinker** in **Aphorisms**... + +--- + +**When to Use:** +- User requests research on specific thinker (Hitchens, Deutsch, Harris, Spinoza, Feynman) +- User says "research [thinker] quotes", "find [author] aphorisms", "what did [thinker] say about [topic]" +- Need to expand database with quotes from key philosophers +- Building out thinker sections in database + +**Prerequisites:** +- Aphorism database exists at `~/.opencode/skills/aphorisms/Database/aphorisms.md` +- Clear understanding of which thinker to research +- Optional: specific theme/topic to focus research on + +--- + +## Workflow Steps + +### Step 1: Identify Research Target + +**Required:** +- **Thinker Name** - Which philosopher/author to research +- Must be one of the five key thinkers OR user-specified author + +**Optional:** +- **Theme Focus** - Specific topic area (e.g., "Hitchens on rationality", "Feynman on doubt") +- **Quote Count** - How many quotes to find (default: 10-15 best quotes) + +**Key Thinkers & Their Focus Areas:** + +**Christopher Hitchens (1949-2011)** +- Focus: Rationality, skepticism, intellectual honesty, evidence-based thinking +- Key Works: God Is Not Great, Hitch-22, Letters to a Young Contrarian +- TELOS Alignment: Truth-seeking, questioning authority, rational discourse + +**David Deutsch (1953-present)** +- Focus: Knowledge creation, optimism, explanations, problem-solving +- Key Works: The Beginning of Infinity, The Fabric of Reality +- TELOS Alignment: Progress through knowledge, problems are solvable, explanatory frameworks + +**Sam Harris (1967-present)** +- Focus: Rationality, meditation, free will, morality, science +- Key Works: The End of Faith, Waking Up, Free Will, The Moral Landscape +- TELOS Alignment: Scientific rationality applied to ethics, mindfulness, evidence-based thinking + +**Baruch Spinoza (1632-1677)** +- Focus: Ethics, reason, freedom through understanding, nature +- Key Works: Ethics, Tractatus Theologico-Politicus +- TELOS Alignment: Rational ethics, freedom through knowledge, understanding reality + +**Richard Feynman (1918-1988)** +- Focus: Curiosity, scientific thinking, doubt as tool, clarity, intellectual honesty +- Key Works: Surely You're Joking Mr. Feynman, The Pleasure of Finding Things Out, The Character of Physical Law +- TELOS Alignment: Curiosity-driven learning, doubt enables knowledge, clarity of thought + +--- + +### Step 2: Comprehensive Research + +**Launch Parallel Research:** + +Use Research Skill with multiple researchers for comprehensive coverage: + +```bash +# Launch 3-5 parallel research agents +# Each focuses on different sources/angles +research_skill.parallel_research( + query="[Thinker Name] most impactful quotes on [theme/general]", + agents=["DeepResearcher", "GeminiResearcher"], + depth="standard" +) +``` + +**Research Sources to Cover:** + +1. **Primary Sources** + - Direct quotes from books, speeches, interviews + - Verified from original works + - Proper attribution to specific source + +2. **Quote Collections** + - Goodreads, BrainyQuote, WikiQuote + - Verify authenticity (many misattributions exist) + - Cross-reference multiple sources + +3. **Academic & Biographical Sources** + - Biographies and scholarly analysis + - Context for when quotes were said + - Understanding thinker's philosophy + +4. **Interviews & Lectures** + - YouTube transcripts (use fabric -y) + - Podcast appearances + - Public talks and debates + +**Research Queries:** + +``` +"[Thinker Name] most famous quotes" +"[Thinker Name] on [theme]" +"[Thinker Name] wisdom about [TELOS topic]" +"[Thinker Name] book quotes [Work Title]" +"[Thinker Name] interview quotes" +"best [Thinker Name] aphorisms" +``` + +--- + +### Step 3: Filter & Verify Quotes + +**Initial Collection:** +- Gather 30-50 potential quotes from research +- Include source attribution for each + +**Filtering Criteria:** + +**1. Authenticity** (CRITICAL) +- Can quote be verified in primary source? +- Is attribution correct (not misattributed)? +- Cross-reference multiple sources +- When in doubt, mark as "Disputed" or skip + +**2. TELOS Alignment** (HIGH PRIORITY) +- Does quote embody wisdom, rationality, truth-seeking? +- Aligns with user's philosophy? +- Relevant to human flourishing and progress? +- Actionable or profound insight? + +**3. Quotability** (HIGH PRIORITY) +- Concise and memorable? +- Stands alone without extensive context? +- Clear meaning without specialized knowledge? +- Impactful phrasing? + +**4. Thematic Relevance** (MEDIUM PRIORITY) +- Fits existing theme categories? +- Relevant to newsletter topics? +- Applicable to modern life? + +**5. Uniqueness** (MEDIUM PRIORITY) +- Offers insight not already well-represented? +- Different angle or perspective? +- Not redundant with existing database quotes? + +**Filter down to top 10-15 quotes:** +- All must pass Authenticity check +- Most should score high on TELOS Alignment and Quotability +- Variety across themes +- Represent thinker's philosophy well + +--- + +### Step 4: Add Context for Each Quote + +**For each selected quote, provide:** + +**1. Source** +- Book title, chapter, page (if available) +- Speech/lecture title and date +- Interview source and date +- Original publication + +**2. Background** +- What was context when said? +- What was thinker addressing? +- Why is this significant? +- How does it represent their philosophy? + +**3. Relevance** +- Why does this matter for the user's audience? +- How does it apply to modern life? +- Connection to TELOS philosophy? +- Practical wisdom it provides? + +**Example:** + +**Quote:** "What can be asserted without evidence can be dismissed without evidence." +**Source:** Christopher Hitchens, "God Is Not Great" (2007) +**Background:** Hitchens' formulation of the burden of proof - often called "Hitchens' Razor." Counters unfalsifiable claims by establishing that extraordinary claims require evidence. Core to his skeptical methodology. +**Relevance:** Essential for critical thinking in age of misinformation. Applies to evaluating AI claims, security threats, market hype - anything requiring evidence-based assessment. + +--- + +### Step 5: Organize by Theme + +**Group quotes by primary theme:** + +- Work Ethic & Excellence +- Resilience & Strength +- Curiosity & Intelligence +- Learning & Education +- Stoicism & Control +- Risk & Action +- Wisdom & Truth-Seeking (if creating new category) +- Rationality & Evidence +- Etc. + +**Within each theme:** +- Order by impact/quotability +- Or chronologically by when said +- Or by source (book order) + +**Purpose:** +- Makes database more useful for find-aphorism workflow +- Shows breadth of thinker's wisdom +- Easier to discover relevant quotes + +--- + +### Step 6: Format for Database + +**Use standard format for each quote:** + +```markdown +**"[Full quote text]"** +- Author: [Thinker Name] +- Theme: [Primary Theme], [Secondary Theme] +- Context: [Source and background - 1-3 sentences] +``` + +**Organize under thinker section:** + +```markdown +### [Thinker Name] + +#### [Theme Category 1] + +**"[Quote 1]"** +- Author: [Name] +- Theme: [Themes] +- Context: [Context] + +**"[Quote 2]"** +- Author: [Name] +- Theme: [Themes] +- Context: [Context] + +#### [Theme Category 2] + +[More quotes...] +``` + +--- + +### Step 7: Add to Database + +**Read current database:** +```bash +Read ~/.opencode/skills/aphorisms/Database/aphorisms.md +``` + +**Locate thinker's section:** +- Find: `### [Thinker Name]` +- Section should exist with placeholder: `*Quotes to be added from research*` + +**Use Edit to replace placeholder:** +```bash +Edit( + file_path=~/.opencode/skills/aphorisms/Database/aphorisms.md, + old_string="### [Thinker Name]\n*Quotes to be added from research*", + new_string="### [Thinker Name]\n\n[Organized quotes with themes and context]" +) +``` + +**Update Theme Index:** +- Add thinker's name to relevant theme categories +- Ensure index reflects new quotes + +--- + +### Step 8: Report Findings + +**Summary for User:** + +```markdown +## [Thinker Name] Research Complete + +**Total Quotes Added:** [N] + +**Quotes by Theme:** +- [Theme 1]: [N] quotes +- [Theme 2]: [N] quotes +- [Theme 3]: [N] quotes + +**Highlight: Top 3 Quotes** + +1. **"[Quote 1]"** + - Theme: [Theme] + - Why Notable: [Reason] + +2. **"[Quote 2]"** + - Theme: [Theme] + - Why Notable: [Reason] + +3. **"[Quote 3]"** + - Theme: [Theme] + - Why Notable: [Reason] + +**Philosophy Summary:** +[2-3 sentences capturing thinker's core philosophy as represented in quotes] + +**TELOS Alignment:** +[How this thinker's wisdom aligns with user's philosophy] + +**Best Use Cases:** +- Newsletter about [topic] → Quote #X +- Newsletter about [topic] → Quote #Y +- General wisdom → Quote #Z + +**Database Stats:** +- Total aphorisms: [N] +- Total from [Thinker]: [N] +- Themes covered: [N] +``` + +--- + +## Advanced Research Techniques + +### Cross-Reference Analysis + +Compare thinker's quotes to existing database: +- Find complementary quotes (different perspectives on same theme) +- Identify gaps in theme coverage +- Discover unique angles thinker provides + +### Chronological Analysis + +Research thinker's evolution: +- Early career quotes vs late career +- How philosophy developed over time +- Most impactful periods + +### Debate & Discussion Mining + +Find quotes from debates/discussions: +- Thinker responding to criticism +- Defending or explaining positions +- Spontaneous wisdom (not just prepared writing) + +--- + +## Special Considerations by Thinker + +### Christopher Hitchens +**Research Tips:** +- Prolific writer - focus on most famous works +- Excellent debater - YouTube debates are goldmines +- Clear, quotable style - many shareable quotes +- Watch for: Skepticism, rational inquiry, intellectual honesty + +**Priority Sources:** +1. God Is Not Great +2. Letters to a Young Contrarian +3. Debates (vs D'Souza, Craig, Blair, etc.) + +### David Deutsch +**Research Tips:** +- Complex ideas - may need context for quotes to land +- Focus on optimism, problem-solving, knowledge creation +- Less traditionally "quotable" - focus on explanatory power +- Watch for: Epistemology, progress, infinite potential + +**Priority Sources:** +1. The Beginning of Infinity +2. The Fabric of Reality +3. Interviews and lectures + +### Sam Harris +**Research Tips:** +- Multiple domains - rationality, meditation, ethics, politics +- Podcast gold - "Making Sense" episodes +- Balance scientific rationality with contemplative wisdom +- Watch for: Evidence-based thinking, mindfulness, moral clarity + +**Priority Sources:** +1. The End of Faith +2. Waking Up +3. Making Sense podcast quotes +4. Debates and talks + +### Baruch Spinoza +**Research Tips:** +- Historical - language may need modernization +- Dense philosophy - extract clearest statements +- Focus on ethics, freedom, reason +- Watch for: Understanding = freedom, rational ethics + +**Priority Sources:** +1. Ethics (primary work) +2. Tractatus Theologico-Politicus +3. Secondary sources explaining Spinoza +4. Scholarly interpretations + +### Richard Feynman +**Research Tips:** +- Extremely quotable - focus on best of best +- Scientific thinking applicable beyond physics +- Humor and humanity alongside rigor +- Watch for: Curiosity, doubt, clarity, honesty + +**Priority Sources:** +1. Surely You're Joking, Mr. Feynman +2. The Pleasure of Finding Things Out +3. The Character of Physical Law +4. Caltech lectures and interviews + +--- + +## Quality Checks + +Before finalizing research: + +- [ ] All quotes verified from primary or reliable sources +- [ ] Attributions confirmed (no misattributions) +- [ ] 10-15 high-quality quotes selected +- [ ] Each quote has context (source, background, relevance) +- [ ] Themes assigned appropriately +- [ ] TELOS alignment clear for all quotes +- [ ] Quotes organized by theme +- [ ] Formatted correctly for database +- [ ] Database updated with quotes +- [ ] Theme index updated +- [ ] Report provided to user with highlights + +--- + +## Edge Cases + +### What if Research Yields No Good Quotes? + +**Reasons:** +1. Thinker doesn't align with TELOS philosophy +2. Thinker's style isn't quotable (dense academic prose) +3. Works not readily accessible + +**Solutions:** +1. Focus on interviews/speeches (more quotable than books) +2. Extract core ideas and find clearest statements +3. Recommend different thinker better aligned + +### What if Too Many Great Quotes? + +**Prioritize:** +1. Most impactful and memorable +2. Best TELOS alignment +3. Variety across themes +4. Can add more in phases + +**Consider:** +- Add top 10-15 now +- Keep research notes for future additions +- Phase 2: Add next 10 quotes later + +### What if Quotes Need Heavy Context? + +**Two approaches:** +1. **Include minimal context** - Let quote stand alone with brief source note +2. **Add extended context** - If understanding requires background + +**Example:** +Spinoza's philosophical language might need more context than Feynman's accessible style. + +--- + +## Integration with Other Workflows + +### After Research Complete + +**Immediately available for:** +- **find-aphorism.md** - New quotes searchable for newsletter matching +- **search-aphorisms.md** - Discoverable via theme/author search + +### Ongoing Research + +**Track research progress:** +- Hitchens: ✅ Complete (15 quotes added) +- Feynman: ✅ Complete (12 quotes added) +- Harris: 🔄 In progress +- Deutsch: ⏳ Planned +- Spinoza: ⏳ Planned + +### Iterative Enhancement + +**Phase 1:** Core quotes from each thinker (10-15) +**Phase 2:** Expand with more quotes (10+ more) +**Phase 3:** Add new thinkers based on newsletter needs + +--- + +## Success Criteria + +**Research succeeds when:** +- 10-15 verified, high-quality quotes added +- All quotes have proper context and themes +- TELOS alignment clear for each +- Thinker's philosophy well-represented +- Quotes immediately useful for newsletter selection +- User understands thinker's contribution + +**Quality indicators:** +- Quotes feel authentic and impactful +- Context helps understand when/how to use +- Theme diversity shows thinker's breadth +- find-aphorism workflow can leverage new quotes effectively + +--- + +## Example Execution + +### User Input +"Research Richard Feynman quotes about curiosity and scientific thinking" + +### Step 1: Identify Target +- **Thinker:** Richard Feynman +- **Focus:** Curiosity, scientific thinking, doubt, learning +- **Target:** 10-15 best quotes + +### Step 2: Research +Launch parallel researchers: +- Search: "Feynman quotes on curiosity" +- Search: "Feynman scientific method quotes" +- Search: "Feynman The Pleasure of Finding Things Out" +- Search: "Feynman doubt and uncertainty" + +Sources: +- Surely You're Joking, Mr. Feynman +- The Pleasure of Finding Things Out (book and speech) +- Caltech lectures +- Interviews + +### Step 3: Filter +Initial: 45 potential quotes +Verified: 28 authentic +TELOS-aligned: 22 +Most quotable: 15 selected + +### Step 4: Add Context +Example: +**"I would rather have questions that can't be answered than answers that can't be questioned."** +- Source: Attributed to Feynman, though exact source debated; consistent with his philosophy +- Background: Feynman's core epistemology - doubt enables knowledge +- Relevance: Essential for scientific thinking, questioning AI hype, avoiding dogma + +### Step 5: Organize by Theme +- Curiosity & Intelligence: 5 quotes +- Learning & Education: 4 quotes +- Wisdom & Truth-Seeking: 3 quotes +- Risk & Action (experimenting): 3 quotes + +### Step 6: Format +```markdown +### Richard Feynman + +#### Curiosity & Intelligence + +**"I would rather have questions that can't be answered than answers that can't be questioned."** +- Author: Richard Feynman +- Theme: Curiosity & Intelligence, Wisdom & Truth-Seeking +- Context: Feynman's epistemology - doubt enables knowledge growth. Essential for scientific thinking and avoiding dogma. + +[14 more quotes...] +``` + +### Step 7: Add to Database +Use Edit to replace placeholder in Feynman section + +### Step 8: Report +```markdown +## Richard Feynman Research Complete + +**Total Quotes Added:** 15 + +**Quotes by Theme:** +- Curiosity & Intelligence: 5 quotes +- Learning & Education: 4 quotes +- Wisdom & Truth-Seeking: 3 quotes +- Risk & Action: 3 quotes + +**Highlight: Top 3 Quotes** + +1. **"I would rather have questions that can't be answered than answers that can't be questioned."** + - Theme: Curiosity & Truth-Seeking + - Why Notable: Captures scientific method essence and intellectual honesty + +[2 more highlights...] + +**Philosophy Summary:** +Feynman embodied curiosity-driven learning combined with radical intellectual honesty. His emphasis on doubt as a tool, pleasure in discovery, and clarity of explanation makes him perfect for TELOS philosophy. + +**TELOS Alignment:** +Perfect alignment with truth-seeking, rationality, and continuous learning. His quotes inspire curiosity while maintaining rigorous standards for evidence and explanation. + +**Best Use Cases:** +- Newsletter about learning → Multiple options +- Newsletter about questioning AI claims → "Questions vs answers" quote +- Newsletter about scientific thinking → Any Feynman quote +``` + +--- + +## Related Workflows + +- **add-aphorism.md** - Individual quote addition (research feeds into this) +- **find-aphorism.md** - Use researched quotes for newsletter recommendations +- **search-aphorisms.md** - Search new quotes by theme + +--- + +**Last Updated:** 2025-11-20 diff --git a/.opencode/skills/Utilities/Aphorisms/Workflows/SearchAphorisms.md b/.opencode/skills/Utilities/Aphorisms/Workflows/SearchAphorisms.md new file mode 100755 index 00000000..2a546999 --- /dev/null +++ b/.opencode/skills/Utilities/Aphorisms/Workflows/SearchAphorisms.md @@ -0,0 +1,720 @@ +# Search Aphorisms + +**Purpose:** Search aphorism database by theme, keyword, author, or topic to discover relevant quotes. + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the SearchAphorisms workflow in the Aphorisms skill to search quotes"}' \ + > /dev/null 2>&1 & +``` + +Running **SearchAphorisms** in **Aphorisms**... + +--- + +**When to Use:** +- User wants to explore quotes on specific theme +- User says "search aphorisms about [topic]", "find quotes on [theme]", "show me [keyword] quotes" +- Browsing database for inspiration +- Finding quotes before knowing exact newsletter theme +- Discovering what's available in database + +**Prerequisites:** +- Aphorism database exists at `~/.opencode/skills/aphorisms/Database/aphorisms.md` +- Search query or theme provided +- Database Read for comprehensive search + +--- + +## Workflow Steps + +### Step 1: Parse Search Query + +**Identify Search Type:** + +**1. Theme Search** +``` +User: "Search aphorisms about resilience" +User: "Find quotes on learning" +User: "Show stoicism quotes" +``` +→ Search by established theme categories + +**2. Keyword Search** +``` +User: "Search for quotes with 'curiosity'" +User: "Find aphorisms mentioning 'fear'" +User: "Quotes about 'action'" +``` +→ Search quote text for specific words + +**3. Author Search** +``` +User: "Show me all Feynman quotes" +User: "What do we have from Hitchens?" +User: "Marcus Aurelius aphorisms" +``` +→ Filter by specific author + +**4. Topic Search** +``` +User: "Quotes about overcoming setbacks" +User: "Aphorisms for newsletter about AI hype" +User: "Wisdom on decision-making" +``` +→ Semantic search across themes and content + +**5. Combination Search** +``` +User: "Feynman quotes about learning" +User: "Stoic quotes on control" +User: "Short quotes about action" +``` +→ Multiple filters applied + +--- + +### Step 2: Read Database + +```bash +Read ~/.opencode/skills/aphorisms/Database/aphorisms.md +``` + +**Load full context:** +- All aphorisms with metadata +- Theme index +- Thinker sections +- Usage history + +--- + +### Step 3: Execute Search + +### Theme Search + +**Match to established themes:** +- Work Ethic & Excellence +- Resilience & Strength +- Fear & Mindset +- Passion & Enthusiasm +- Competition & Progress +- Curiosity & Intelligence +- Investment & Self-Development +- Present Moment & Enjoyment +- Learning & Education +- Stoicism & Control +- Risk & Action +- Adversity + +**Process:** +1. Identify which theme(s) match search query +2. May match multiple themes (e.g., "growth" matches Resilience + Learning) +3. Extract all quotes tagged with matching theme(s) + +**Example:** +``` +Query: "Search aphorisms about resilience" + +Matching Themes: Resilience & Strength, Adversity + +Results: +- Bob Marley (strength through necessity) +- Rocky Balboa (getting hit and moving forward) +- Muhammad Ali (suffer now, champion later) +- Unknown (pain → strength) +``` + +--- + +### Keyword Search + +**Search quote text directly:** +``` +For each aphorism in database: + If keyword appears in quote text: + Add to results +``` + +**Case-insensitive:** +- "curiosity" matches "Curiosity", "CURIOSITY", "curious" + +**Partial matches:** +- "learn" matches "learning", "learned", "learner" + +**Context awareness:** +- Also check author names and contexts for keyword + +**Example:** +``` +Query: "Find aphorisms mentioning 'fear'" + +Results: +- Robert Heller: "Fear is excitement without breath" +- (Any other quotes with "fear" in text or context) +``` + +--- + +### Author Search + +**Filter by author name:** +``` +For each aphorism in database: + If author matches search: + Add to results +``` + +**Flexible matching:** +- "Feynman" matches "Richard Feynman" +- "Marcus Aurelius" matches "Aurelius" +- "Ali" matches "Muhammad Ali" + +**Result organization:** +- Group by theme if author has multiple quotes +- Chronological if known +- By source if documented + +**Example:** +``` +Query: "Show me all Marcus Aurelius quotes" + +Results: +Marcus Aurelius (1 quote): + +**Stoicism & Control:** +- "You have power over your mind - not outside events. Realize this, and you will find strength." +``` + +--- + +### Topic Search (Semantic) + +**More complex - requires understanding:** + +**Process:** +1. Analyze topic semantically +2. Identify related themes +3. Look for keyword variations +4. Check quote contexts for relevance + +**Example:** +``` +Query: "Quotes about overcoming setbacks" + +Analysis: +- Primary themes: Resilience, Adversity, Strength +- Related concepts: Persistence, growth, difficulty +- Keywords: setback, challenge, overcome, persist, adversity + +Search: +1. Themes: Resilience & Strength, Adversity +2. Keywords: "setback", "challenge", "adversity", "difficulty" +3. Context: Any mention of overcoming challenges + +Results: [Combined theme + keyword matches] +``` + +--- + +### Combination Search + +**Apply multiple filters:** + +**Example:** +``` +Query: "Feynman quotes about learning" + +Filters: +1. Author = Feynman +2. Theme = Learning & Education OR Curiosity & Intelligence + +Process: +- Get all Feynman quotes +- Filter for those tagged with learning/curiosity themes +- Return intersection + +Results: [Feynman quotes specifically about learning] +``` + +--- + +### Step 4: Rank Results + +**Ranking Criteria:** + +**1. Relevance Score (0-10)** +- Exact theme match: 10 +- Keyword in quote: 8-10 +- Keyword in context: 5-7 +- Related theme: 4-6 +- Tangential: 1-3 + +**2. Quotability (0-10)** +- Clear, concise: 8-10 +- Needs some context: 5-7 +- Complex or long: 3-4 + +**3. Freshness (0-10)** +- Never used: 10 +- Used >6 months ago: 7-9 +- Used recently: 3-6 +- Used very recently: 0-2 + +**Total Score:** Sum (max 30) + +**Sort results:** +- Highest score first +- Break ties by quotability +- Secondary sort by author diversity + +--- + +### Step 5: Format Results + +**Standard Format:** + +```markdown +## Search Results: "[Query]" + +**Found [N] matching aphorisms** + +--- + +### Result 1: [Quote Summary/First Few Words] + +**"[Full quote text]"** + +**Author:** [Author Name] +**Themes:** [Theme 1], [Theme 2] +**Context:** [Brief context] + +**Why This Matches:** +[Specific relevance to search query] + +**Usage:** [Last used: date / Never used] + +**Score:** [X/30] + +--- + +### Result 2: [Quote Summary] + +[Same format...] + +--- + +[All results...] + +--- + +## Summary + +**Total Results:** [N] +**By Theme:** +- [Theme 1]: [N] quotes +- [Theme 2]: [N] quotes + +**By Author:** +- [Author 1]: [N] quotes +- [Author 2]: [N] quotes + +**Top Recommendation:** [Result #1 title] +``` + +--- + +### Step 6: Present Options + +**Provide user with:** +1. Full results formatted +2. Summary statistics +3. Top recommendation +4. Related searches suggestions + +**Related Searches:** +```markdown +## You Might Also Like + +Based on your search for "[Query]", you might also be interested in: +- [Related theme 1] +- [Related theme 2] +- Quotes from [Related author] +``` + +--- + +## Advanced Search Features + +### Exclude Filter + +``` +User: "Search resilience quotes but not Rocky Balboa" + +Process: +1. Get all resilience quotes +2. Filter out Rocky Balboa quotes +3. Return remaining results +``` + +### Recency Filter + +``` +User: "Show me unused quotes about learning" + +Process: +1. Get all learning quotes +2. Filter for usage = never OR >6 months +3. Return fresh quotes +``` + +### Length Filter + +``` +User: "Short quotes about action" + +Process: +1. Get all action quotes +2. Filter for character count < 150 +3. Return concise options +``` + +### Source Filter + +``` +User: "Quotes from books, not movies" + +Process: +1. Check context/source metadata +2. Filter for literary sources +3. Exclude film quotes +``` + +--- + +## Search by Use Case + +### Newsletter Topic Matching + +``` +User: "I'm writing about AI safety. What quotes do we have?" + +Process: +1. Analyze topic: AI safety = caution, wisdom, responsibility, careful thinking +2. Map to themes: Stoicism & Control, Wisdom & Truth-Seeking, Risk & Action (cautious side) +3. Search across multiple themes +4. Rank by relevance to "careful, wise decision-making" + +Results: +- Marcus Aurelius (control what you can) +- Confucius (learning + thinking balance) +- Feynman (if available - on scientific rigor) +``` + +### Mood Matching + +``` +User: "I need an inspiring quote" + +Process: +1. Identify "inspiring" mood +2. Themes: Excellence, Resilience, Progress, Action +3. Filter for positive, empowering tone +4. Avoid darker or cautionary quotes + +Results: +- Tim Grover, Muhammad Ali, Robin Sharma quotes +- Focus on achievement and growth +``` + +### Audience Matching + +``` +User: "Quote for technical audience" + +Process: +1. Identify technical audience preference +2. Favor: Feynman, Deutsch, scientific thinkers +3. Avoid: Pop culture references, sports figures +4. Emphasize: Rationality, evidence, rigorous thinking + +Results: +- Feynman quotes on curiosity and doubt +- Deutsch on knowledge creation +- Harris on rationality +``` + +--- + +## Quality Checks + +Before returning results: + +- [ ] All results genuinely match search query +- [ ] Relevance scores accurately reflect match quality +- [ ] Results ranked appropriately +- [ ] Variety in authors if possible +- [ ] Context explains why each quote matches +- [ ] Usage history noted for freshness +- [ ] Related searches are actually relevant +- [ ] Formatting is clear and scannable + +--- + +## Edge Cases + +### No Results Found + +```markdown +## Search Results: "[Query]" + +**No matching aphorisms found.** + +**Suggestions:** +1. **Broaden search** - Try related themes: + - [Related theme 1] + - [Related theme 2] + +2. **Research new quotes** - Would you like me to research quotes about "[topic]"? + +3. **Similar searches that would work:** + - [Alternative query 1] + - [Alternative query 2] + +**Available themes in database:** +- [List all themes] +``` + +### Too Many Results + +``` +Query: "Search for quotes" +(No specific criteria - matches everything) + +Response: +"Your search is very broad. Database contains [N] total aphorisms. + +Would you like to: +1. Browse by theme (show theme index) +2. Browse by author (show author list) +3. Search by specific topic (ask: what topic?) +4. Show random sampling (10 diverse quotes)" +``` + +### Ambiguous Query + +``` +Query: "Search for quotes about change" + +Issue: "Change" could mean: +- Personal transformation (growth) +- Taking action (doing something different) +- Accepting reality (stoic acceptance) +- Progress and innovation + +Response: +"'Change' can mean several things. Which are you most interested in? +1. Personal growth and transformation +2. Taking action and making things happen +3. Accepting reality and what you can't control +4. Progress and innovation + +Or should I show results from all categories?" +``` + +### Contradictory Search + +``` +Query: "Stoic quotes about taking big risks" + +Issue: Stoicism emphasizes control and caution; big risks are opposite + +Response: +"Stoicism and big risk-taking are somewhat contradictory philosophies. + +Would you prefer: +1. Stoic quotes about calculated risk (wisdom + action) +2. Risk-taking quotes from non-stoic sources +3. Quotes balancing wisdom with action + +Or explain what you're looking for and I'll find the best match?" +``` + +--- + +## Integration with Other Workflows + +### After Search Results + +**Next Actions:** + +**1. Use in newsletter:** +- Results feed directly into find-aphorism.md analysis +- User can request: "Use result #3 for my newsletter" + +**2. Expand research:** +- If user likes specific author: "Research more [Author] quotes" +- Uses research-thinker.md workflow + +**3. Add new quotes:** +- If search reveals gap: "Research quotes about [missing theme]" +- Uses research-thinker.md then add-aphorism.md + +--- + +## Saved Searches + +**Track common searches:** +- Newsletter topics frequently covered +- Most-searched themes +- Recurring patterns + +**Optimize database:** +- Expand themes that get searched often +- Research thinkers for frequently requested topics + +--- + +## Success Criteria + +**Search succeeds when:** +- Results accurately match user query +- User finds relevant quote(s) from results +- Ranking makes sense (best matches first) +- Variety in results when possible +- Clear explanation of why each quote matches +- User can quickly decide which quote to use + +**Quality indicators:** +- User says "perfect, exactly what I needed" +- User selects from results without requesting more searches +- Relevance scores align with user perception +- Results are diverse and interesting + +--- + +## Example Execution + +### User Input +"Search aphorisms about learning and curiosity" + +### Step 1: Parse Query +- **Type:** Theme search + Keyword +- **Themes:** Learning & Education, Curiosity & Intelligence +- **Keywords:** "learn", "learning", "curious", "curiosity" + +### Step 2: Read Database +Load full database context + +### Step 3: Execute Search + +**Theme matches:** +- Curiosity & Intelligence theme: Aaron Swartz, Dorothy Parker (if added), Einstein (if added) +- Learning & Education theme: Gandhi, Krishnamurti, Confucius + +**Keyword matches:** +- Any quotes with "learn", "curious", "curiosity" in text + +**Results:** +1. Aaron Swartz (curiosity = intelligence) +2. Gandhi (live/learn) +3. Krishnamurti (lifelong learning) +4. Confucius (learning + thinking) +5. Einstein (if added - questioning and curiosity) + +### Step 4: Rank Results + +**Scores:** +1. Aaron Swartz - 28/30 (exact theme match + keyword + never used) +2. Einstein - 26/30 (exact theme + keyword + highly quotable) +3. Krishnamurti - 24/30 (theme match + keyword + context) +4. Confucius - 23/30 (theme match + keyword) +5. Gandhi - 22/30 (theme match + indirect keyword) + +### Step 5: Format Results + +```markdown +## Search Results: "learning and curiosity" + +**Found 5 matching aphorisms** + +--- + +### Result 1: Curiosity = Intelligence + +**"Be curious. Read widely. Try new things. What people call intelligence just boils down to curiosity."** + +**Author:** Aaron Swartz +**Themes:** Curiosity & Intelligence +**Context:** Internet activist defining intelligence as active curiosity + +**Why This Matches:** +Direct hit on both "curiosity" and learning-oriented message. Reframes intelligence as curiosity in action. + +**Usage:** Never used + +**Score:** 28/30 + +--- + +### Result 2: Never Stop Questioning + +**"The important thing is not to stop questioning. Curiosity has its own reason for existing."** + +**Author:** Albert Einstein +**Themes:** Curiosity & Intelligence, Learning & Education +**Context:** From 1955 interview "Old Man's Advice to Youth" + +**Why This Matches:** +Directly mentions questioning and curiosity. Emphasizes intrinsic value of curiosity. + +**Usage:** Never used + +**Score:** 26/30 + +--- + +[Results 3-5 formatted similarly...] + +--- + +## Summary + +**Total Results:** 5 +**By Theme:** +- Curiosity & Intelligence: 2 quotes +- Learning & Education: 4 quotes (some overlap) + +**By Author:** +- Swartz: 1 +- Einstein: 1 +- Krishnamurti: 1 +- Confucius: 1 +- Gandhi: 1 + +**Top Recommendation:** Aaron Swartz (Result #1) + +## You Might Also Like + +Based on your search for "learning and curiosity", you might also be interested in: +- Feynman quotes (when added - perfect match for this theme) +- Quotes about "intellectual honesty" +- Quotes about "questioning assumptions" +``` + +### Step 6: Present +User reviews results and selects Aaron Swartz quote for newsletter + +--- + +## Related Workflows + +- **find-aphorism.md** - Use search results for newsletter matching +- **research-thinker.md** - Expand database if search reveals gaps +- **add-aphorism.md** - Add discovered quotes during research + +--- + +**Last Updated:** 2025-11-20 diff --git a/.opencode/skills/Utilities/Browser/README.md b/.opencode/skills/Utilities/Browser/README.md new file mode 100755 index 00000000..9b07aa35 --- /dev/null +++ b/.opencode/skills/Utilities/Browser/README.md @@ -0,0 +1,230 @@ +# Browser - Code-First Interface + +**Replace the token-heavy Playwright MCP with direct code execution.** + +## Why? + +| Approach | Tokens | Performance | +|----------|--------|-------------| +| Playwright MCP | ~13,700 at load | MCP protocol overhead | +| Code-first | ~50-200 per op | Direct Playwright API | +| **Savings** | **99%+** | Faster execution | + +## Quick Start + +```bash +# Install dependencies +cd ~/.opencode/skills/Browser +bun install + +# Take a screenshot +bun examples/screenshot.ts https://example.com + +# Verify a page loads +bun examples/verify-page.ts https://example.com +``` + +## Usage + +### Basic + +```typescript +import { PlaywrightBrowser } from '~/.opencode/skills/Browser/index.ts' + +const browser = new PlaywrightBrowser() +await browser.launch() +await browser.navigate('https://example.com') +await browser.screenshot({ path: 'screenshot.png' }) +await browser.close() +``` + +### Form Interaction + +```typescript +const browser = new PlaywrightBrowser() +await browser.launch({ headless: false }) // Watch it work + +await browser.navigate('https://example.com/login') +await browser.fill('#email', 'test@example.com') +await browser.fill('#password', 'secret') +await browser.click('button[type="submit"]') +await browser.waitForNavigation() + +const title = await browser.getTitle() +console.log(`Logged in! Page: ${title}`) + +await browser.close() +``` + +### Page Verification + +```typescript +const browser = new PlaywrightBrowser() +await browser.launch() + +await browser.navigate('https://example.com') + +// Check specific element exists +await browser.waitForSelector('h1') +const heading = await browser.getVisibleText('h1') +console.log(`Found heading: ${heading}`) + +// Check for console errors +const errors = browser.getConsoleLogs({ type: 'error' }) +if (errors.length > 0) { + console.log('Console errors:', errors) +} + +// Get accessibility tree (like MCP uses) +const a11yTree = await browser.getAccessibilityTree() + +await browser.close() +``` + +### Device Emulation + +```typescript +const browser = new PlaywrightBrowser() +await browser.launch() + +// Emulate iPhone +await browser.setDevice('iPhone 14') +await browser.navigate('https://example.com') +await browser.screenshot({ path: 'mobile.png' }) + +// Or set custom viewport +await browser.resize(375, 812) + +await browser.close() +``` + +## API Reference + +### Constructor + +```typescript +const browser = new PlaywrightBrowser() +``` + +### Launch Options + +```typescript +await browser.launch({ + browser: 'chromium', // 'chromium' | 'firefox' | 'webkit' + headless: true, // false to see browser + viewport: { width: 1280, height: 720 }, + userAgent: 'Custom UA' +}) +``` + +### Navigation + +| Method | Description | +|--------|-------------| +| `navigate(url, options?)` | Go to URL | +| `goBack()` | Browser back | +| `goForward()` | Browser forward | +| `reload()` | Refresh page | +| `getUrl()` | Current URL | +| `getTitle()` | Page title | + +### Capture + +| Method | Description | +|--------|-------------| +| `screenshot(options?)` | Take screenshot | +| `getVisibleText(selector?)` | Extract text | +| `getVisibleHtml(options?)` | Get HTML | +| `savePdf(path, options?)` | Export PDF | +| `getAccessibilityTree()` | A11y snapshot | + +### Interaction + +| Method | Description | +|--------|-------------| +| `click(selector)` | Click element | +| `hover(selector)` | Mouse hover | +| `fill(selector, value)` | Fill input | +| `type(selector, text, delay?)` | Type with delay | +| `select(selector, value)` | Select dropdown | +| `pressKey(key, selector?)` | Keyboard | +| `drag(source, target)` | Drag and drop | +| `uploadFile(selector, path)` | File upload | + +### Waiting + +| Method | Description | +|--------|-------------| +| `waitForSelector(selector)` | Wait for element | +| `waitForNavigation()` | Wait for page | +| `waitForNetworkIdle()` | Wait for idle | +| `wait(ms)` | Fixed delay | + +### JavaScript + +| Method | Description | +|--------|-------------| +| `evaluate(script)` | Run JS | +| `getConsoleLogs()` | Console output | +| `setUserAgent(ua)` | Change UA | + +### iFrame + +| Method | Description | +|--------|-------------| +| `iframeClick(iframe, el)` | Click in iframe | +| `iframeFill(iframe, el, val)` | Fill in iframe | + +## Token Savings + +The key insight: Playwright MCP loads ~13,700 tokens of tool definitions at startup. This code-first approach: + +1. **Zero startup cost** - No tokens until you use a function +2. **~50-200 tokens per operation** - Just the code you execute +3. **Full Playwright API** - Not limited to MCP's 21 tools + +### Example Calculation + +**Scenario:** Take 5 screenshots during a session + +| Approach | Tokens | +|----------|--------| +| MCP (loaded at start) | 13,700 | +| Code-first (5 × ~100) | 500 | +| **Savings** | 96.4% | + +## Migration from MCP + +### Before (MCP) + +```json +{ + "mcpServers": { + "playwright": { + "command": "npx", + "args": ["@playwright/mcp@latest"] + } + } +} +``` + +### After (Code-First) + +```typescript +// Just import and use - no MCP server needed +import { PlaywrightBrowser } from '~/.opencode/skills/Browser/index.ts' + +const browser = new PlaywrightBrowser() +// ... use it +``` + +## Requirements + +- Bun runtime +- Playwright (`bun add playwright`) + +## Related + +- [File-Based MCP Architecture](~/.opencode/skills/CORE/SYSTEM/DOCUMENTATION/FileBasedMCPs.md) +- [Apify Code-First](../Apify/README.md) +- [Playwright Docs](https://playwright.dev) diff --git a/.opencode/skills/Utilities/Browser/SKILL.md b/.opencode/skills/Utilities/Browser/SKILL.md new file mode 100755 index 00000000..3b6d5d30 --- /dev/null +++ b/.opencode/skills/Utilities/Browser/SKILL.md @@ -0,0 +1,268 @@ +--- +name: Browser +description: Browser automation with debug visibility. USE WHEN browser, screenshot, debug web, verify UI. +version: 2.0.0 +--- + +## Customization + +**Before executing, check for user customizations at:** +`~/.opencode/skills/CORE/USER/SKILLCUSTOMIZATIONS/Browser/` + +If this directory exists, load and apply any PREFERENCES.md, configurations, or resources found there. These override default behavior. If the directory does not exist, proceed with skill defaults. + +# Browser v2.0.0 - Debug-First Browser Automation + +**Debugging visibility by DEFAULT.** Console logs, network requests, and errors are always captured. No opt-in required. + +--- + +## Philosophy + +Debugging shouldn't be opt-in. Like good logging frameworks - you don't turn on logging when you have a problem, you have it enabled from the start so the data exists when problems occur. + +**Headless by default.** All automation runs in headless mode. When the user says "show me" or wants to see what the assistant is seeing, open the URL in their preferred browser from `~/.opencode/skills/CORE/USER/TECHSTACKPREFERENCES.md`: + +```bash +# Open URL in user's preferred browser +open -a "$BROWSER" "" # BROWSER from tech stack prefs +``` + +**v2.0.0 Changes:** +- Session auto-starts on first use (no explicit `session start`) +- Console and network capture always enabled +- Diagnostic output included by default +- 30-minute idle timeout (auto-cleanup) +- New primary command: `Browse.ts ` for full diagnostics + +--- + +## Quick Start + +```bash +# Navigate with full diagnostics (PRIMARY COMMAND) +bun run ~/.opencode/skills/Browser/Tools/Browse.ts https://example.com + +# Output: +# 📸 Screenshot: /tmp/browse-1704614400.png +# 🔴 Console Errors (2): ... +# 🌐 Failed Requests (1): ... +# 📊 Network: 34 requests | 1.2MB | avg 120ms +# ✅ Page: "Example" loaded successfully +``` + +Session auto-starts. No setup needed. + +--- + +## Commands + +### Primary - Navigate with Diagnostics + +```bash +bun run Browse.ts +``` + +Navigates to URL, takes screenshot, and reports: +- Console errors and warnings +- Failed network requests (4xx, 5xx) +- Network statistics +- Page load status + +### Query Commands + +```bash +bun run Browse.ts errors # Console errors only +bun run Browse.ts warnings # Console warnings only +bun run Browse.ts console # All console output +bun run Browse.ts network # All network activity +bun run Browse.ts failed # Failed requests only (4xx, 5xx) +``` + +### Interaction Commands + +```bash +bun run Browse.ts click # Click element +bun run Browse.ts fill # Fill input +bun run Browse.ts type # Type with delay +bun run Browse.ts screenshot [path] # Take screenshot +bun run Browse.ts navigate # Navigate without report +bun run Browse.ts eval "" # Execute JavaScript +bun run Browse.ts open # Open in user's preferred browser (from tech stack prefs) +``` + +### Management Commands + +```bash +bun run Browse.ts status # Session info +bun run Browse.ts restart # Fresh session (clears logs) +bun run Browse.ts stop # Stop session (rarely needed) +``` + +--- + +## Debugging Workflow + +**Scenario: "Why isn't the user list loading?"** + +```bash +# Step 1: Load the page +$ bun run Browse.ts https://myapp.com/users + +📸 Screenshot: /tmp/browse-xxx.png + +🔴 Console Errors (1): + • TypeError: Cannot read property 'map' of undefined + +🌐 Failed Requests (1): + • GET /api/users → 500 Internal Server Error + +📊 Network: 23 requests | 847KB | avg 89ms +⚠️ Page: "Users" loaded with issues +``` + +**Immediately know:** +1. API returning 500 +2. Frontend JS crashing because no data +3. Specific error location + +**Step 2: Dig deeper** + +```bash +# Full console output +$ bun run Browse.ts console + +# All network activity +$ bun run Browse.ts network + +# Just the failures +$ bun run Browse.ts failed +``` + +--- + +## Architecture + +### Auto-Start Session + +First command auto-starts a persistent browser session: + +``` +Any command → ensureSession() → Session running? + ├─ Yes → Execute command + └─ No → Start session → Execute command +``` + +No explicit `session start` needed. + +### Always-On Event Capture + +From the moment the browser launches: +- **Console logs** - All `console.log`, `console.error`, etc. +- **Network requests** - Every request with headers, timing, size +- **Network responses** - Status codes, response times, sizes +- **Page errors** - Uncaught exceptions, promise rejections + +### Idle Timeout + +Session auto-closes after 30 minutes of inactivity: +- No zombie processes +- No manual cleanup needed +- Restarts automatically on next command + +--- + +## Comparison to v1.x + +| Aspect | v1.x | v2.0.0 | +|--------|------|--------| +| Session management | Manual `session start/stop` | Automatic | +| Console capture | Only in session mode | Always | +| Network capture | Only in session mode | Always | +| Error visibility | Must query explicitly | Default in output | +| Idle cleanup | Manual stop | Auto 30-min timeout | +| Primary command | `screenshot ` | `` (full diagnostic) | + +--- + +## API Reference + +### CLI Tool + +**Location:** `~/.opencode/skills/Browser/Tools/Browse.ts` + +| Command | Description | +|---------|-------------| +| `` | Navigate with full diagnostics | +| `errors` | Show console errors | +| `warnings` | Show console warnings | +| `console` | Show all console output | +| `network` | Show network activity | +| `failed` | Show failed requests | +| `screenshot [path]` | Take screenshot | +| `navigate ` | Navigate without report | +| `click ` | Click element | +| `fill ` | Fill input | +| `type ` | Type with delay | +| `eval ""` | Execute JavaScript | +| `open ` | Open in user's preferred browser (tech stack prefs) | +| `status` | Session info | +| `restart` | Fresh session | +| `stop` | Stop session | + +### Server Endpoints + +**Location:** `~/.opencode/skills/Browser/Tools/BrowserSession.ts` + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/diagnostics` | GET | Full diagnostic summary | +| `/console` | GET | Console logs | +| `/network` | GET | Network logs | +| `/health` | GET | Health check | +| `/session` | GET | Session info | +| `/navigate` | POST | Navigate (clears logs) | +| `/click` | POST | Click element | +| `/fill` | POST | Fill input | +| `/screenshot` | POST | Take screenshot | +| `/evaluate` | POST | Run JavaScript | +| `/stop` | POST | Stop server | + +--- + +## TypeScript API + +For complex automation, use the TypeScript API directly: + +```typescript +import { PlaywrightBrowser } from '~/.opencode/skills/Browser/index.ts' + +const browser = new PlaywrightBrowser() +await browser.launch({ headless: true }) +await browser.navigate('https://example.com') + +// Get diagnostics +const errors = browser.getConsoleLogs({ type: 'error' }) +const failed = browser.getNetworkLogs({ status: [400, 404, 500] }) +const stats = browser.getNetworkStats() + +await browser.close() +``` + +**Full API:** See `index.ts` for all methods. + +--- + +## VERIFY Phase Integration + +**MANDATORY for verifying web changes:** + +```bash +# Before claiming any web change is "live" or "working": +bun run Browse.ts https://example.com/changed-page + +# Check the screenshot AND diagnostics +# If errors or failed requests exist, the change is NOT verified +``` + +**If you haven't LOOKED at the rendered page and its diagnostics, you CANNOT claim it works.** diff --git a/.opencode/skills/Utilities/Browser/Tools/Browse.ts b/.opencode/skills/Utilities/Browser/Tools/Browse.ts new file mode 100755 index 00000000..2ce0a89f --- /dev/null +++ b/.opencode/skills/Utilities/Browser/Tools/Browse.ts @@ -0,0 +1,694 @@ +#!/usr/bin/env bun +/** + * Browse CLI Tool v2.0.0 - Debug-First Browser Automation + * + * Browser automation with debugging visibility by DEFAULT. + * Console logs, network requests, and errors are always captured. + * + * Usage: + * bun run Browse.ts # Navigate with full diagnostics + * bun run Browse.ts errors # Show console errors + * bun run Browse.ts network # Show network activity + * bun run Browse.ts failed # Show failed requests (4xx, 5xx) + * bun run Browse.ts screenshot [path] # Take screenshot of current page + * bun run Browse.ts click # Click element + * bun run Browse.ts fill # Fill input field + * + * Session auto-starts on first use. No explicit start needed. + */ + +import { PlaywrightBrowser } from '../index.ts' + +const VOICE_SERVER = 'http://localhost:8888/notify' +const STATE_FILE = '/tmp/browser-session.json' +const DEFAULT_PORT = 9222 +const SESSION_TIMEOUT = 5000 // 5s to wait for session start +const SETTINGS_PATH = `${process.env.HOME}/.opencode/settings.json` + +// ============================================ +// SETTINGS +// ============================================ + +interface Settings { + techStack?: { + browser?: string + terminal?: string + packageManager?: string + devServerPort?: number + } +} + +async function getSettings(): Promise { + try { + const file = Bun.file(SETTINGS_PATH) + if (await file.exists()) { + return JSON.parse(await file.text()) + } + } catch {} + return {} +} + +async function getBrowser(): Promise { + const settings = await getSettings() + return settings.techStack?.browser || 'Dia' +} + +// ============================================ +// TYPES +// ============================================ + +interface SessionState { + pid: number + port: number + sessionId: string + startedAt: string + headless: boolean + url: string +} + +interface Diagnostics { + errors: Array<{ type: string; text: string; timestamp: number }> + warnings: Array<{ type: string; text: string; timestamp: number }> + failedRequests: Array<{ + url: string + method: string + status: number + statusText?: string + }> + stats: { + totalRequests: number + totalResponses: number + totalSize: number + avgDuration: number + } + pageTitle: string + pageUrl: string +} + +// ============================================ +// UTILITIES +// ============================================ + +async function notify(message: string): Promise { + try { + await fetch(VOICE_SERVER, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ message }) + }) + } catch { + // Voice server not running - silent fail + } +} + +function formatBytes(bytes: number): string { + if (bytes === 0) return '0 B' + const k = 1024 + const sizes = ['B', 'KB', 'MB', 'GB'] + const i = Math.floor(Math.log(bytes) / Math.log(k)) + return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i] +} + +function truncate(str: string, maxLen: number): string { + if (str.length <= maxLen) return str + return str.slice(0, maxLen - 3) + '...' +} + +// ============================================ +// SESSION MANAGEMENT +// ============================================ + +async function getSessionState(): Promise { + try { + const file = Bun.file(STATE_FILE) + if (await file.exists()) { + const content = await file.text() + if (content.trim()) { + return JSON.parse(content) + } + } + } catch {} + return null +} + +async function isSessionRunning(): Promise { + const state = await getSessionState() + if (!state) return false + + try { + const res = await fetch(`http://localhost:${state.port}/health`, { + signal: AbortSignal.timeout(1000) + }) + return res.ok + } catch { + // Server not responding - clean up orphan state + try { + const fs = await import('fs/promises') + await fs.unlink(STATE_FILE) + } catch {} + return false + } +} + +async function ensureSession(): Promise { + // Check if already running + const state = await getSessionState() + if (state) { + try { + const res = await fetch(`http://localhost:${state.port}/health`, { + signal: AbortSignal.timeout(1000) + }) + if (res.ok) { + return state.port + } + } catch {} + } + + // Need to start session + const port = DEFAULT_PORT + + // Check port availability + try { + const res = await fetch(`http://localhost:${port}/health`, { + signal: AbortSignal.timeout(500) + }) + if (res.ok) { + return port // Already running on this port + } + } catch {} + + // Start server in background + const { spawn } = await import('child_process') + const serverPath = new URL('./BrowserSession.ts', import.meta.url).pathname + + const env: Record = { + ...process.env as Record, + BROWSER_PORT: String(port), + BROWSER_HEADLESS: 'true' + } + + const child = spawn('bun', ['run', serverPath], { + detached: true, + stdio: 'ignore', + env + }) + child.unref() + + // Wait for server to be ready + for (let i = 0; i < 30; i++) { + await Bun.sleep(200) + try { + const res = await fetch(`http://localhost:${port}/health`, { + signal: AbortSignal.timeout(1000) + }) + if (res.ok) { + return port + } + } catch {} + } + + throw new Error('Failed to start browser session') +} + +async function sessionCommand( + endpoint: string, + body?: any, + method = 'POST' +): Promise { + const port = await ensureSession() + + const options: RequestInit = { + method, + signal: AbortSignal.timeout(60000) // 60s for long operations + } + + if (body && method !== 'GET') { + options.headers = { 'Content-Type': 'application/json' } + options.body = JSON.stringify(body) + } + + const url = method === 'GET' && body + ? `http://localhost:${port}/${endpoint}?${new URLSearchParams(body)}` + : `http://localhost:${port}/${endpoint}` + + const res = await fetch(url, options) + const data = await res.json() as { success: boolean; data?: any; error?: string } + + if (data.success) { + return data.data + } else { + throw new Error(data.error || 'Unknown error') + } +} + +// ============================================ +// DIAGNOSTIC FORMATTING +// ============================================ + +function formatDiagnostics( + diag: Diagnostics, + screenshotPath?: string +): string { + const lines: string[] = [] + + // Screenshot (if taken) + if (screenshotPath) { + lines.push(`📸 Screenshot: ${screenshotPath}`) + lines.push('') + } + + // Console errors + if (diag.errors.length > 0) { + lines.push(`🔴 Console Errors (${diag.errors.length}):`) + for (const err of diag.errors.slice(0, 5)) { + lines.push(` • ${truncate(err.text, 100)}`) + } + if (diag.errors.length > 5) { + lines.push(` ... and ${diag.errors.length - 5} more`) + } + lines.push('') + } + + // Console warnings + if (diag.warnings.length > 0) { + lines.push(`⚠️ Console Warnings (${diag.warnings.length}):`) + for (const warn of diag.warnings.slice(0, 3)) { + lines.push(` • ${truncate(warn.text, 100)}`) + } + if (diag.warnings.length > 3) { + lines.push(` ... and ${diag.warnings.length - 3} more`) + } + lines.push('') + } + + // Failed requests + if (diag.failedRequests.length > 0) { + lines.push(`🌐 Failed Requests (${diag.failedRequests.length}):`) + for (const req of diag.failedRequests.slice(0, 5)) { + const urlPath = new URL(req.url).pathname + lines.push(` • ${req.method} ${truncate(urlPath, 50)} → ${req.status} ${req.statusText || ''}`) + } + if (diag.failedRequests.length > 5) { + lines.push(` ... and ${diag.failedRequests.length - 5} more`) + } + lines.push('') + } + + // Network summary + lines.push(`📊 Network: ${diag.stats.totalRequests} requests | ${formatBytes(diag.stats.totalSize)} | avg ${Math.round(diag.stats.avgDuration)}ms`) + + // Final status + const hasIssues = diag.errors.length > 0 || diag.failedRequests.length > 0 + if (hasIssues) { + lines.push(`⚠️ Page: "${diag.pageTitle}" loaded with issues`) + } else { + lines.push(`✅ Page: "${diag.pageTitle}" loaded successfully`) + } + + return lines.join('\n') +} + +// ============================================ +// COMMANDS +// ============================================ + +async function debugUrl(url: string): Promise { + // Navigate + await sessionCommand('navigate', { url }) + + // Take screenshot + const timestamp = Date.now() + const screenshotPath = `/tmp/browse-${timestamp}.png` + await sessionCommand('screenshot', { path: screenshotPath }) + + // Get diagnostics + const diag = await sessionCommand('diagnostics', {}, 'GET') as Diagnostics + + // Output + console.log(formatDiagnostics(diag, screenshotPath)) +} + +async function showErrors(): Promise { + const diag = await sessionCommand('diagnostics', {}, 'GET') as Diagnostics + + if (diag.errors.length === 0) { + console.log('✅ No console errors') + return + } + + console.log(`🔴 Console Errors (${diag.errors.length}):\n`) + for (const err of diag.errors) { + const time = new Date(err.timestamp).toLocaleTimeString() + console.log(`[${time}] ${err.text}\n`) + } +} + +async function showWarnings(): Promise { + const diag = await sessionCommand('diagnostics', {}, 'GET') as Diagnostics + + if (diag.warnings.length === 0) { + console.log('✅ No console warnings') + return + } + + console.log(`⚠️ Console Warnings (${diag.warnings.length}):\n`) + for (const warn of diag.warnings) { + const time = new Date(warn.timestamp).toLocaleTimeString() + console.log(`[${time}] ${warn.text}\n`) + } +} + +async function showConsole(): Promise { + const logs = await sessionCommand('console', {}, 'GET') as Array<{ + type: string + text: string + timestamp: number + }> + + if (logs.length === 0) { + console.log('📋 No console output') + return + } + + console.log(`📋 Console Output (${logs.length} entries):\n`) + for (const log of logs) { + const time = new Date(log.timestamp).toLocaleTimeString() + const icon = log.type === 'error' ? '🔴' : + log.type === 'warning' ? '⚠️' : + log.type === 'info' ? 'ℹ️' : ' ' + console.log(`${icon} [${time}] ${log.text}`) + } +} + +async function showNetwork(): Promise { + const logs = await sessionCommand('network', {}, 'GET') as Array<{ + type: string + url: string + method: string + status?: number + duration?: number + size?: number + }> + + if (logs.length === 0) { + console.log('🌐 No network activity') + return + } + + // Show only responses (more useful) + const responses = logs.filter(l => l.type === 'response') + + console.log(`🌐 Network Activity (${responses.length} responses):\n`) + for (const log of responses.slice(-20)) { + const urlPath = new URL(log.url).pathname + const status = log.status || 0 + const icon = status >= 400 ? '❌' : status >= 300 ? '↪️' : '✅' + const size = log.size ? formatBytes(log.size) : '' + const duration = log.duration ? `${log.duration}ms` : '' + console.log(`${icon} ${status} ${log.method} ${truncate(urlPath, 50)} ${size} ${duration}`) + } +} + +async function showFailed(): Promise { + const diag = await sessionCommand('diagnostics', {}, 'GET') as Diagnostics + + if (diag.failedRequests.length === 0) { + console.log('✅ No failed requests') + return + } + + console.log(`🌐 Failed Requests (${diag.failedRequests.length}):\n`) + for (const req of diag.failedRequests) { + console.log(`❌ ${req.status} ${req.method} ${req.url}`) + } +} + +async function takeScreenshot(path?: string): Promise { + const screenshotPath = path || `/tmp/screenshot-${Date.now()}.png` + await sessionCommand('screenshot', { path: screenshotPath }) + console.log(`📸 Screenshot: ${screenshotPath}`) +} + +async function navigate(url: string): Promise { + const result = await sessionCommand('navigate', { url }) + console.log(`Navigated to: ${result.url}`) +} + +async function click(selector: string): Promise { + await sessionCommand('click', { selector }) + console.log(`Clicked: ${selector}`) +} + +async function fill(selector: string, value: string): Promise { + await sessionCommand('fill', { selector, value }) + console.log(`Filled: ${selector}`) +} + +async function type(selector: string, text: string): Promise { + await sessionCommand('type', { selector, text }) + console.log(`Typed in: ${selector}`) +} + +async function evaluate(script: string): Promise { + const result = await sessionCommand('evaluate', { script }) + console.log(JSON.stringify(result.result, null, 2)) +} + +async function showStatus(): Promise { + try { + const state = await getSessionState() + if (!state) { + console.log('No session running') + return + } + + const session = await sessionCommand('session', {}, 'GET') + console.log('Browser Session:') + console.log(` ID: ${session.sessionId}`) + console.log(` Port: ${session.port}`) + console.log(` URL: ${session.url || '(none)'}`) + console.log(` Title: ${session.title || '(none)'}`) + console.log(` Started: ${session.startedAt}`) + console.log(` Idle timeout: ${session.idleTimeout}`) + } catch { + console.log('Session not responding') + } +} + +async function restart(): Promise { + // Stop if running + try { + const state = await getSessionState() + if (state) { + await fetch(`http://localhost:${state.port}/stop`, { + method: 'POST', + signal: AbortSignal.timeout(2000) + }) + await Bun.sleep(500) + } + } catch {} + + // Clean up state file + try { + const fs = await import('fs/promises') + await fs.unlink(STATE_FILE) + } catch {} + + // Start fresh + await ensureSession() + console.log('Session restarted') +} + +async function stop(): Promise { + const state = await getSessionState() + if (!state) { + console.log('No session running') + return + } + + try { + await fetch(`http://localhost:${state.port}/stop`, { + method: 'POST', + signal: AbortSignal.timeout(2000) + }) + console.log('Session stopped') + } catch { + console.log('Session already stopped') + } +} + +async function openUrl(url: string): Promise { + // Use browser from settings.json techStack + const browser = await getBrowser() + const { spawn } = await import('child_process') + spawn('open', ['-a', browser, url], { detached: true, stdio: 'ignore' }).unref() + console.log(`Opened in ${browser}: ${url}`) +} + +// ============================================ +// MAIN +// ============================================ + +function showHelp(): void { + console.log(` +Browse CLI v2.0.0 - Debug-First Browser Automation + +Usage: + bun run Browse.ts Navigate with full diagnostics + bun run Browse.ts errors Show console errors + bun run Browse.ts warnings Show console warnings + bun run Browse.ts console Show all console output + bun run Browse.ts network Show network activity + bun run Browse.ts failed Show failed requests (4xx, 5xx) + bun run Browse.ts screenshot [path] Take screenshot of current page + bun run Browse.ts navigate Navigate without full report + bun run Browse.ts click Click element + bun run Browse.ts fill Fill input field + bun run Browse.ts type Type with delay + bun run Browse.ts eval "" Execute JavaScript + bun run Browse.ts open Open in user's browser + bun run Browse.ts status Show session info + bun run Browse.ts restart Restart session (clear state) + bun run Browse.ts stop Stop session + +Session auto-starts on first use. No explicit start needed. + +Examples: + bun run Browse.ts https://example.com + bun run Browse.ts errors + bun run Browse.ts click "#submit" + bun run Browse.ts fill "#email" "test@example.com" + `) +} + +async function main(): Promise { + const args = process.argv.slice(2) + const command = args[0] + + if (!command || command === 'help' || command === '--help' || command === '-h') { + showHelp() + return + } + + try { + // URL detection - if it looks like a URL, treat as debug command + if (command.startsWith('http://') || command.startsWith('https://') || command.startsWith('localhost')) { + const url = command.startsWith('localhost') ? `http://${command}` : command + await debugUrl(url) + return + } + + // Named commands + switch (command) { + case 'errors': + await showErrors() + break + + case 'warnings': + await showWarnings() + break + + case 'console': + await showConsole() + break + + case 'network': + await showNetwork() + break + + case 'failed': + await showFailed() + break + + case 'screenshot': + await takeScreenshot(args[1]) + break + + case 'navigate': + if (!args[1]) { + console.error('URL required') + process.exit(1) + } + await navigate(args[1]) + break + + case 'click': + if (!args[1]) { + console.error('Selector required') + process.exit(1) + } + await click(args[1]) + break + + case 'fill': + if (!args[1] || !args[2]) { + console.error('Selector and value required') + process.exit(1) + } + await fill(args[1], args[2]) + break + + case 'type': + if (!args[1] || !args[2]) { + console.error('Selector and text required') + process.exit(1) + } + await type(args[1], args[2]) + break + + case 'eval': + if (!args[1]) { + console.error('JavaScript code required') + process.exit(1) + } + await evaluate(args[1]) + break + + case 'open': + if (!args[1]) { + console.error('URL required') + process.exit(1) + } + await openUrl(args[1]) + break + + case 'status': + await showStatus() + break + + case 'restart': + await restart() + break + + case 'stop': + await stop() + break + + // Legacy session commands (redirect to new interface) + case 'session': + const subCmd = args[1] + if (subCmd === 'start') { + await ensureSession() + console.log('Session started (auto-starts on any command now)') + } else if (subCmd === 'stop') { + await stop() + } else if (subCmd === 'status') { + await showStatus() + } else { + console.log('Session commands deprecated. Session auto-starts on first use.') + console.log('Use: Browse.ts | errors | network | failed | etc.') + } + break + + default: + console.error(`Unknown command: ${command}`) + console.log('Run with --help for usage') + process.exit(1) + } + } catch (err: any) { + console.error(`Error: ${err.message}`) + process.exit(1) + } +} + +main() diff --git a/.opencode/skills/Utilities/Browser/Tools/BrowserSession.ts b/.opencode/skills/Utilities/Browser/Tools/BrowserSession.ts new file mode 100755 index 00000000..4218a5d2 --- /dev/null +++ b/.opencode/skills/Utilities/Browser/Tools/BrowserSession.ts @@ -0,0 +1,445 @@ +#!/usr/bin/env bun +/** + * Browser Session Server v2.0.0 - Debug-First Persistent Browser + * + * Persistent Playwright browser with ALWAYS-ON event capture. + * Console logs, network requests, and errors captured from launch. + * + * Usage: + * # Started automatically by Browse.ts (not directly) + * BROWSER_PORT=9222 bun run BrowserSession.ts + * + * New API (v2.0.0): + * GET /diagnostics - Full diagnostic summary (errors, warnings, failed requests) + * GET /console - All console logs + * GET /network - All network activity + * + * Standard API: + * GET /health - Server health check + * GET /session - Current session info + * POST /navigate - Navigate to URL (clears logs for fresh page) + * POST /click - Click element + * POST /fill - Fill input + * POST /screenshot - Take screenshot + * GET /text - Get visible text + * POST /evaluate - Run JavaScript + * POST /stop - Stop server + */ + +import { PlaywrightBrowser } from '../index.ts' + +const CONFIG = { + port: parseInt(process.env.BROWSER_PORT || '9222'), + headless: process.env.BROWSER_HEADLESS === 'true', + viewport: { + width: parseInt(process.env.BROWSER_WIDTH || '1920'), + height: parseInt(process.env.BROWSER_HEIGHT || '1080') + }, + stateFile: '/tmp/browser-session.json', + idleTimeout: 30 * 60 * 1000 // 30 minutes +} + +const browser = new PlaywrightBrowser() +const sessionId = crypto.randomUUID().slice(0, 8) +const startedAt = new Date().toISOString() +let lastActivity = Date.now() + +// ============================================ +// STATE MANAGEMENT +// ============================================ + +async function saveState(): Promise { + try { + const state = { + pid: process.pid, + port: CONFIG.port, + sessionId, + startedAt, + headless: CONFIG.headless, + url: browser.getUrl() + } + await Bun.write(CONFIG.stateFile, JSON.stringify(state, null, 2)) + } catch (error) { + console.error('Failed to save state:', error) + } +} + +async function cleanup(): Promise { + console.log('\nShutting down browser session...') + try { + await browser.close() + } catch {} + try { + const file = Bun.file(CONFIG.stateFile) + if (await file.exists()) { + await Bun.write(CONFIG.stateFile, '') + const fs = await import('fs/promises') + await fs.unlink(CONFIG.stateFile) + } + } catch {} + console.log('Session closed.') + process.exit(0) +} + +// ============================================ +// IDLE TIMEOUT +// ============================================ + +function checkIdleTimeout(): void { + const idle = Date.now() - lastActivity + if (idle > CONFIG.idleTimeout) { + console.log(`Idle timeout (${Math.round(idle / 60000)} minutes) - shutting down`) + cleanup() + } +} + +// Check every minute +setInterval(checkIdleTimeout, 60 * 1000) + +// ============================================ +// RESPONSE HELPERS +// ============================================ + +function json(data: any, status = 200): Response { + return new Response(JSON.stringify(data), { + status, + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, POST, DELETE, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type' + } + }) +} + +function success(data?: any): Response { + return json({ success: true, data }) +} + +function error(message: string, status = 500): Response { + return json({ success: false, error: message }, status) +} + +// ============================================ +// LAUNCH BROWSER +// ============================================ + +console.log('Starting browser session...') +console.log(` Port: ${CONFIG.port}`) +console.log(` Headless: ${CONFIG.headless}`) +console.log(` Viewport: ${CONFIG.viewport.width}x${CONFIG.viewport.height}`) +console.log(` Idle timeout: ${CONFIG.idleTimeout / 60000} minutes`) + +await browser.launch({ + headless: CONFIG.headless, + viewport: CONFIG.viewport +}) + +// ============================================ +// HTTP SERVER +// ============================================ + +const server = Bun.serve({ + port: CONFIG.port, + + async fetch(req) { + const url = new URL(req.url) + const method = req.method + + // Update activity timestamp on every request + lastActivity = Date.now() + + // CORS preflight + if (method === 'OPTIONS') { + return new Response(null, { + headers: { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, POST, DELETE, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type' + } + }) + } + + try { + // ======================================== + // DIAGNOSTIC ENDPOINTS (NEW in v2.0.0) + // ======================================== + + // Full diagnostics - errors, warnings, failed requests, stats + if (url.pathname === '/diagnostics' && method === 'GET') { + const allLogs = browser.getConsoleLogs() + const errors = allLogs.filter(l => l.type === 'error') + const warnings = allLogs.filter(l => l.type === 'warning') + + const networkLogs = browser.getNetworkLogs({ type: 'response' }) + const failedRequests = networkLogs + .filter(l => l.status && l.status >= 400) + .map(l => ({ + url: l.url, + method: l.method, + status: l.status!, + statusText: l.statusText + })) + + const stats = browser.getNetworkStats() + + return success({ + errors, + warnings, + failedRequests, + stats, + pageTitle: await browser.getTitle(), + pageUrl: browser.getUrl() + }) + } + + // Console logs + if (url.pathname === '/console' && method === 'GET') { + const type = url.searchParams.get('type') as any + const limit = parseInt(url.searchParams.get('limit') || '100') + const logs = browser.getConsoleLogs({ type: type || undefined, limit }) + return success(logs) + } + + // Network logs + if (url.pathname === '/network' && method === 'GET') { + const limit = parseInt(url.searchParams.get('limit') || '100') + const logs = browser.getNetworkLogs({ limit }) + return success(logs) + } + + // ======================================== + // STANDARD ENDPOINTS + // ======================================== + + // Health check + if (url.pathname === '/health' && method === 'GET') { + return success({ + status: 'ok', + sessionId, + uptime: Date.now() - new Date(startedAt).getTime() + }) + } + + // Session info + if (url.pathname === '/session' && method === 'GET') { + return success({ + sessionId, + startedAt, + port: CONFIG.port, + headless: CONFIG.headless, + url: browser.getUrl(), + title: await browser.getTitle(), + idleTimeout: `${CONFIG.idleTimeout / 60000} minutes`, + lastActivity: new Date(lastActivity).toISOString() + }) + } + + // Navigate - CLEARS LOGS for fresh page diagnostics + if (url.pathname === '/navigate' && method === 'POST') { + const body = await req.json() + if (!body.url) return error('url required', 400) + + // Clear logs before navigating for clean diagnostic slate + browser.getConsoleLogs({ clear: true }) + browser.clearNetworkLogs() + + await browser.navigate(body.url, { waitUntil: body.waitUntil || 'networkidle' }) + await saveState() + + return success({ + url: browser.getUrl(), + title: await browser.getTitle() + }) + } + + // Click + if (url.pathname === '/click' && method === 'POST') { + const body = await req.json() + if (!body.selector) return error('selector required', 400) + await browser.click(body.selector, { timeout: body.timeout }) + return success({ clicked: body.selector }) + } + + // Fill + if (url.pathname === '/fill' && method === 'POST') { + const body = await req.json() + if (!body.selector || body.value === undefined) return error('selector and value required', 400) + await browser.fill(body.selector, body.value) + return success({ filled: body.selector }) + } + + // Type (character by character) + if (url.pathname === '/type' && method === 'POST') { + const body = await req.json() + if (!body.selector || !body.text) return error('selector and text required', 400) + await browser.type(body.selector, body.text, body.delay) + return success({ typed: body.selector }) + } + + // Screenshot + if (url.pathname === '/screenshot' && method === 'POST') { + const body = await req.json() + const path = body.path || '/tmp/screenshot.png' + await browser.screenshot({ + path, + fullPage: body.fullPage || false, + selector: body.selector + }) + return success({ path }) + } + + // Get visible text + if (url.pathname === '/text' && method === 'GET') { + const selector = url.searchParams.get('selector') || undefined + const text = await browser.getVisibleText(selector) + return success({ text }) + } + + // Get HTML + if (url.pathname === '/html' && method === 'GET') { + const selector = url.searchParams.get('selector') || undefined + const html = await browser.getVisibleHtml({ selector }) + return success({ html }) + } + + // Evaluate JavaScript + if (url.pathname === '/evaluate' && method === 'POST') { + const body = await req.json() + if (!body.script) return error('script required', 400) + const result = await browser.evaluate(body.script) + return success({ result }) + } + + // Wait for selector + if (url.pathname === '/wait' && method === 'POST') { + const body = await req.json() + if (!body.selector) return error('selector required', 400) + await browser.waitForSelector(body.selector, { + state: body.state, + timeout: body.timeout + }) + return success({ found: body.selector }) + } + + // Wait for text + if (url.pathname === '/wait-text' && method === 'POST') { + const body = await req.json() + if (!body.text) return error('text required', 400) + await browser.waitForText(body.text, { + state: body.state, + timeout: body.timeout + }) + return success({ found: body.text }) + } + + // Hover + if (url.pathname === '/hover' && method === 'POST') { + const body = await req.json() + if (!body.selector) return error('selector required', 400) + await browser.hover(body.selector) + return success({ hovered: body.selector }) + } + + // Press key + if (url.pathname === '/press' && method === 'POST') { + const body = await req.json() + if (!body.key) return error('key required', 400) + await browser.pressKey(body.key, body.selector) + return success({ pressed: body.key }) + } + + // Select dropdown + if (url.pathname === '/select' && method === 'POST') { + const body = await req.json() + if (!body.selector || !body.value) return error('selector and value required', 400) + await browser.select(body.selector, body.value) + return success({ selected: body.value }) + } + + // Tabs - list + if (url.pathname === '/tabs' && method === 'GET') { + const tabs = browser.getTabs() + return success({ tabs }) + } + + // Tabs - new + if (url.pathname === '/tabs' && method === 'POST') { + const body = await req.json() + await browser.newTab(body.url) + return success({ created: true, url: body.url }) + } + + // Tabs - close + if (url.pathname.startsWith('/tabs/') && method === 'DELETE') { + const index = parseInt(url.pathname.split('/')[2]) + if (isNaN(index)) return error('invalid tab index', 400) + await browser.switchTab(index) + await browser.closeTab() + return success({ closed: index }) + } + + // Tabs - switch + if (url.pathname.startsWith('/tabs/') && method === 'POST') { + const index = parseInt(url.pathname.split('/')[2]) + if (isNaN(index)) return error('invalid tab index', 400) + await browser.switchTab(index) + return success({ switched: index }) + } + + // Reload + if (url.pathname === '/reload' && method === 'POST') { + await browser.reload() + return success({ reloaded: true }) + } + + // Go back + if (url.pathname === '/back' && method === 'POST') { + await browser.goBack() + return success({ back: true }) + } + + // Go forward + if (url.pathname === '/forward' && method === 'POST') { + await browser.goForward() + return success({ forward: true }) + } + + // Resize viewport + if (url.pathname === '/resize' && method === 'POST') { + const body = await req.json() + if (!body.width || !body.height) return error('width and height required', 400) + await browser.resize(body.width, body.height) + return success({ width: body.width, height: body.height }) + } + + // Stop server + if (url.pathname === '/stop' && method === 'POST') { + setTimeout(() => cleanup(), 100) + return success({ stopping: true }) + } + + return error('Not found', 404) + + } catch (err: any) { + console.error('Request error:', err.message) + return error(err.message) + } + } +}) + +await saveState() +console.log(`\nBrowser session started!`) +console.log(` Session ID: ${sessionId}`) +console.log(` URL: http://localhost:${CONFIG.port}`) +console.log(` Diagnostics: http://localhost:${CONFIG.port}/diagnostics`) +console.log(`\nSession will auto-close after ${CONFIG.idleTimeout / 60000} minutes of inactivity.`) +console.log(`Press Ctrl+C to stop manually.`) + +// Cleanup handlers +process.on('SIGTERM', cleanup) +process.on('SIGINT', cleanup) +process.on('uncaughtException', (err) => { + console.error('Uncaught exception:', err) + cleanup() +}) diff --git a/.opencode/skills/Utilities/Browser/Workflows/Extract.md b/.opencode/skills/Utilities/Browser/Workflows/Extract.md new file mode 100755 index 00000000..f1323a40 --- /dev/null +++ b/.opencode/skills/Utilities/Browser/Workflows/Extract.md @@ -0,0 +1,125 @@ +# Extract Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the Extract workflow in the Browser skill to extract page content"}' \ + > /dev/null 2>&1 & +``` + +Running **Extract** in **Browser**... + +--- + +Extract content from web pages. + +## Steps + +1. **Launch and navigate** + ```typescript + import { PlaywrightBrowser } from '~/.opencode/skills/Browser/index.ts' + const browser = new PlaywrightBrowser() + await browser.launch({ headless: true }) + await browser.navigate(url) + ``` + +2. **Extract content** + ```typescript + // Text content + const text = await browser.getVisibleText() + + // Specific element + const heading = await browser.getVisibleText('h1') + + // HTML + const html = await browser.getVisibleHtml({ + selector: 'article', + removeScripts: true, + removeStyles: true, + minify: true + }) + ``` + +3. **Close browser** + ```typescript + await browser.close() + ``` + +## Extraction Methods + +### Get All Visible Text +```typescript +const allText = await browser.getVisibleText() +``` + +### Get Element Text +```typescript +const title = await browser.getVisibleText('h1') +const articles = await browser.getVisibleText('.article-list') +``` + +### Get Clean HTML +```typescript +const html = await browser.getVisibleHtml({ + selector: 'main', + removeScripts: true, + removeStyles: true, + removeComments: true, + minify: true +}) +``` + +### Run JavaScript +```typescript +const data = await browser.evaluate(() => { + return { + title: document.title, + links: Array.from(document.querySelectorAll('a')).map(a => a.href), + timestamp: Date.now() + } +}) +``` + +### Get Accessibility Tree +```typescript +const a11y = await browser.getAccessibilityTree() +// Structured data about page elements +``` + +## Example: Article Extraction + +```typescript +import { PlaywrightBrowser } from '~/.opencode/skills/Browser/index.ts' + +const browser = new PlaywrightBrowser() +await browser.launch({ headless: true }) + +await browser.navigate('https://example.com/article') + +// Wait for content to load +await browser.waitForSelector('article') + +// Extract +const title = await browser.getVisibleText('h1') +const content = await browser.getVisibleText('article') +const html = await browser.getVisibleHtml({ + selector: 'article', + removeScripts: true +}) + +console.log({ title, contentLength: content.length }) + +await browser.close() +``` + +## Example: Save as PDF + +```typescript +await browser.navigate('https://example.com/report') +await browser.savePdf('/tmp/report.pdf', { + format: 'A4', + printBackground: true +}) +``` diff --git a/.opencode/skills/Utilities/Browser/Workflows/Interact.md b/.opencode/skills/Utilities/Browser/Workflows/Interact.md new file mode 100755 index 00000000..6868637b --- /dev/null +++ b/.opencode/skills/Utilities/Browser/Workflows/Interact.md @@ -0,0 +1,115 @@ +# Interact Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the Interact workflow in the Browser skill to interact with elements"}' \ + > /dev/null 2>&1 & +``` + +Running **Interact** in **Browser**... + +--- + +Fill forms, click buttons, and interact with page elements. + +## Steps + +1. **Launch and navigate** + ```typescript + import { PlaywrightBrowser } from '~/.opencode/skills/Browser/index.ts' + const browser = new PlaywrightBrowser() + await browser.launch({ headless: false }) // Watch it work + await browser.navigate(url) + ``` + +2. **Interact with elements** + ```typescript + await browser.fill('#email', 'test@example.com') + await browser.fill('#password', 'secret') + await browser.click('button[type="submit"]') + ``` + +3. **Wait for result** + ```typescript + await browser.waitForNavigation() + // or + await browser.waitForSelector('.success-message') + ``` + +4. **Verify and close** + ```typescript + const result = await browser.getVisibleText('.result') + await browser.close() + ``` + +## Common Interactions + +### Click +```typescript +await browser.click('button') +await browser.click('#submit-btn') +await browser.click('a[href="/login"]') +``` + +### Fill Input +```typescript +await browser.fill('input[name="email"]', 'test@example.com') +await browser.fill('#search', 'query') +``` + +### Type with Delay (Realistic) +```typescript +await browser.type('#search', 'query', 50) // 50ms between keys +``` + +### Select Dropdown +```typescript +await browser.select('#country', 'US') +await browser.select('#tags', ['tag1', 'tag2']) // Multi-select +``` + +### Press Keys +```typescript +await browser.pressKey('Enter') +await browser.pressKey('Escape') +await browser.pressKey('Tab', '#input-field') // On specific element +``` + +### Hover +```typescript +await browser.hover('.dropdown-trigger') +await browser.click('.dropdown-item') // Now visible +``` + +### Drag and Drop +```typescript +await browser.drag('#draggable', '#drop-zone') +``` + +### File Upload +```typescript +await browser.uploadFile('input[type="file"]', '/path/to/file.pdf') +``` + +## Example: Login Flow + +```typescript +import { PlaywrightBrowser } from '~/.opencode/skills/Browser/index.ts' + +const browser = new PlaywrightBrowser() +await browser.launch({ headless: false }) + +await browser.navigate('https://example.com/login') +await browser.fill('#email', 'user@example.com') +await browser.fill('#password', 'password123') +await browser.click('button[type="submit"]') + +await browser.waitForNavigation() +const welcomeText = await browser.getVisibleText('.welcome-message') +console.log(`Logged in: ${welcomeText}`) + +await browser.close() +``` diff --git a/.opencode/skills/Utilities/Browser/Workflows/Screenshot.md b/.opencode/skills/Utilities/Browser/Workflows/Screenshot.md new file mode 100755 index 00000000..843b781e --- /dev/null +++ b/.opencode/skills/Utilities/Browser/Workflows/Screenshot.md @@ -0,0 +1,71 @@ +# Screenshot Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the Screenshot workflow in the Browser skill to capture screenshots"}' \ + > /dev/null 2>&1 & +``` + +Running **Screenshot** in **Browser**... + +--- + +Take a screenshot of a URL. + +## Steps + +1. **Launch browser** + ```typescript + import { PlaywrightBrowser } from '~/.opencode/skills/Browser/index.ts' + const browser = new PlaywrightBrowser() + await browser.launch({ headless: true }) + ``` + +2. **Navigate to URL** + ```typescript + await browser.navigate(url) + ``` + +3. **Take screenshot** + ```typescript + await browser.screenshot({ + path: '/tmp/screenshot.png', + fullPage: false // true for full page + }) + ``` + +4. **Close browser** + ```typescript + await browser.close() + ``` + +## Options + +| Option | Default | Description | +|--------|---------|-------------| +| `path` | - | Output file path | +| `fullPage` | false | Capture full scrollable page | +| `selector` | - | Capture specific element | +| `type` | 'png' | 'png' or 'jpeg' | +| `quality` | - | JPEG quality (0-100) | + +## Example: Element Screenshot + +```typescript +await browser.screenshot({ + selector: '.hero-section', + path: '/tmp/hero.png' +}) +``` + +## Example: Mobile Screenshot + +```typescript +await browser.launch() +await browser.setDevice('iPhone 14') +await browser.navigate(url) +await browser.screenshot({ path: '/tmp/mobile.png' }) +``` diff --git a/.opencode/skills/Utilities/Browser/Workflows/Update.md b/.opencode/skills/Utilities/Browser/Workflows/Update.md new file mode 100755 index 00000000..05ab7122 --- /dev/null +++ b/.opencode/skills/Utilities/Browser/Workflows/Update.md @@ -0,0 +1,86 @@ +# Update Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the Update workflow in the Browser skill to sync capabilities"}' \ + > /dev/null 2>&1 & +``` + +Running **Update** in **Browser**... + +--- + +Sync Browser FileMCP with official Playwright MCP capabilities. + +## When to Use + +- Monthly capability check +- After Playwright MCP releases new version +- If browser commands fail unexpectedly + +## Official Source + +**Repository:** [microsoft/playwright-mcp](https://github.com/microsoft/playwright-mcp) +**Package:** `@playwright/mcp` + +## Steps + +### 1. Fetch Latest API Surface + +```bash +# Get Playwright MCP README (contains all tools) +curl -s https://raw.githubusercontent.com/microsoft/playwright-mcp/main/README.md > /tmp/playwright-mcp-readme.md + +# Check npm package version +npm info @playwright/mcp version +``` + +### 2. Extract Tool Definitions + +Official Playwright MCP tools (core): +- browser_click, browser_close, browser_console_messages +- browser_drag, browser_evaluate, browser_file_upload +- browser_handle_dialog, browser_hover, browser_navigate +- browser_navigate_back, browser_network_requests, browser_press_key +- browser_resize, browser_select_option, browser_snapshot +- browser_take_screenshot, browser_type, browser_wait_for +- browser_tabs, browser_pdf_save + +### 3. Compare with FileMCP + +| Official Tool | FileMCP Method | Status | +|---------------|----------------|--------| +| browser_click | `click()` | Implemented | +| browser_screenshot | `screenshot()` | Implemented | +| browser_navigate | `navigate()` | Implemented | +| browser_evaluate | `evaluate()` | Implemented | +| browser_fill_form | `fill()` | Implemented | +| (Vision tools) | - | Not needed | +| (Tracing) | - | Nice-to-have | + +### 4. Update FileMCP if Needed + +For missing critical functionality: +1. Add method to `index.ts` +2. Add CLI command to `Tools/Browse.ts` +3. Update SKILL.md documentation + +### 5. Test + +```bash +# Verify basic operations +bun run ~/.opencode/skills/Browser/Tools/Browse.ts screenshot https://example.com /tmp/test.png +bun run ~/.opencode/skills/Browser/Tools/Browse.ts verify https://example.com "body" +``` + +## Version Tracking + +``` +# Last sync: 2026-01-03 +# Playwright MCP: @playwright/mcp@latest +# FileMCP coverage: 90%+ of core tools +# Known gaps: Vision tools, Testing assertions, Tracing +``` diff --git a/.opencode/skills/Utilities/Browser/Workflows/VerifyPage.md b/.opencode/skills/Utilities/Browser/Workflows/VerifyPage.md new file mode 100755 index 00000000..0e673125 --- /dev/null +++ b/.opencode/skills/Utilities/Browser/Workflows/VerifyPage.md @@ -0,0 +1,94 @@ +# VerifyPage Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the VerifyPage workflow in the Browser skill to verify page loads"}' \ + > /dev/null 2>&1 & +``` + +Running **VerifyPage** in **Browser**... + +--- + +Verify a page loads correctly and check for errors. + +## Steps + +1. **Launch and navigate** + ```typescript + import { PlaywrightBrowser } from '~/.opencode/skills/Browser/index.ts' + const browser = new PlaywrightBrowser() + await browser.launch({ headless: true }) + + const startTime = Date.now() + await browser.navigate(url) + const loadTime = Date.now() - startTime + ``` + +2. **Check basic info** + ```typescript + const title = await browser.getTitle() + const finalUrl = browser.getUrl() // Check for redirects + ``` + +3. **Verify specific element (optional)** + ```typescript + await browser.waitForSelector(selector, { timeout: 5000 }) + const text = await browser.getVisibleText(selector) + ``` + +4. **Check for console errors** + ```typescript + const errors = browser.getConsoleLogs({ type: 'error' }) + if (errors.length > 0) { + console.log('Console errors:', errors) + } + ``` + +5. **Take verification screenshot** + ```typescript + await browser.screenshot({ path: '/tmp/verify.png' }) + ``` + +6. **Close browser** + ```typescript + await browser.close() + ``` + +## Example: Full Verification + +```typescript +import { PlaywrightBrowser } from '~/.opencode/skills/Browser/index.ts' + +const browser = new PlaywrightBrowser() +await browser.launch({ headless: true }) + +await browser.navigate('https://example.com') + +// Basic checks +const title = await browser.getTitle() +console.log(`Title: ${title}`) + +// Check specific element +await browser.waitForSelector('h1') +const heading = await browser.getVisibleText('h1') +console.log(`H1: ${heading}`) + +// Console errors +const errors = browser.getConsoleLogs({ type: 'error' }) +console.log(`Errors: ${errors.length}`) + +// Evidence +await browser.screenshot({ path: '/tmp/verify.png' }) + +await browser.close() +``` + +## CLI Usage + +```bash +bun ~/.opencode/skills/Browser/examples/verify-page.ts https://example.com +``` diff --git a/.opencode/skills/Utilities/Browser/bun.lock b/.opencode/skills/Utilities/Browser/bun.lock new file mode 100755 index 00000000..aaac2ef6 --- /dev/null +++ b/.opencode/skills/Utilities/Browser/bun.lock @@ -0,0 +1,30 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "@kai/playwright", + "dependencies": { + "playwright": "^1.49.0", + }, + "devDependencies": { + "@types/bun": "^1.1.0", + }, + }, + }, + "packages": { + "@types/bun": ["@types/bun@1.3.5", "", { "dependencies": { "bun-types": "1.3.5" } }, "sha512-RnygCqNrd3srIPEWBd5LFeUYG7plCoH2Yw9WaZGyNmdTEei+gWaHqydbaIRkIkcbXwhBT94q78QljxN0Sk838w=="], + + "@types/node": ["@types/node@25.0.3", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA=="], + + "bun-types": ["bun-types@1.3.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-inmAYe2PFLs0SUbFOWSVD24sg1jFlMPxOjOSSCYqUgn4Hsc3rDc7dFvfVYjFPNHtov6kgUeulV4SxbuIV/stPw=="], + + "fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], + + "playwright": ["playwright@1.57.0", "", { "dependencies": { "playwright-core": "1.57.0" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-ilYQj1s8sr2ppEJ2YVadYBN0Mb3mdo9J0wQ+UuDhzYqURwSoW4n1Xs5vs7ORwgDGmyEh33tRMeS8KhdkMoLXQw=="], + + "playwright-core": ["playwright-core@1.57.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ=="], + + "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], + } +} diff --git a/.opencode/skills/Utilities/Browser/examples/comprehensive-test.ts b/.opencode/skills/Utilities/Browser/examples/comprehensive-test.ts new file mode 100755 index 00000000..c624b8b2 --- /dev/null +++ b/.opencode/skills/Utilities/Browser/examples/comprehensive-test.ts @@ -0,0 +1,417 @@ +#!/usr/bin/env bun +/** + * Comprehensive test of all Playwright FileMCP functionality + */ + +import { PlaywrightBrowser } from '../index.ts' + +const TEST_URL = 'https://httpbin.org' +const FORM_URL = 'https://httpbin.org/forms/post' + +interface TestResult { + name: string + status: 'PASS' | 'FAIL' | 'SKIP' + error?: string + duration?: number +} + +const results: TestResult[] = [] + +async function test(name: string, fn: () => Promise): Promise { + const start = Date.now() + try { + await fn() + results.push({ name, status: 'PASS', duration: Date.now() - start }) + console.log(`✅ ${name}`) + } catch (error) { + const msg = error instanceof Error ? error.message : String(error) + results.push({ name, status: 'FAIL', error: msg, duration: Date.now() - start }) + console.log(`❌ ${name}: ${msg}`) + } +} + +function skip(name: string, reason: string): void { + results.push({ name, status: 'SKIP', error: reason }) + console.log(`⏭️ ${name}: ${reason}`) +} + +async function main() { + console.log('=== Playwright FileMCP Comprehensive Test ===\n') + + const browser = new PlaywrightBrowser() + + // ============================================ + // LIFECYCLE + // ============================================ + console.log('\n--- LIFECYCLE ---') + + await test('launch()', async () => { + await browser.launch({ headless: true }) + }) + + // ============================================ + // NAVIGATION + // ============================================ + console.log('\n--- NAVIGATION ---') + + await test('navigate()', async () => { + await browser.navigate(TEST_URL) + }) + + await test('getUrl()', async () => { + const url = browser.getUrl() + if (!url.includes('httpbin.org')) throw new Error(`Expected httpbin.org, got ${url}`) + }) + + await test('getTitle()', async () => { + const title = await browser.getTitle() + if (!title) throw new Error('Title is empty') + }) + + await test('navigate() to form page', async () => { + await browser.navigate(FORM_URL) + }) + + await test('goBack()', async () => { + await browser.goBack() + const url = browser.getUrl() + if (!url.includes('httpbin.org')) throw new Error('goBack failed') + }) + + await test('goForward()', async () => { + await browser.goForward() + const url = browser.getUrl() + if (!url.includes('forms/post')) throw new Error('goForward failed') + }) + + await test('reload()', async () => { + await browser.reload() + }) + + // ============================================ + // CAPTURE + // ============================================ + console.log('\n--- CAPTURE ---') + + await test('screenshot()', async () => { + const buffer = await browser.screenshot() + if (buffer.length < 1000) throw new Error('Screenshot too small') + }) + + await test('screenshot({ fullPage: true })', async () => { + const buffer = await browser.screenshot({ fullPage: true }) + if (buffer.length < 1000) throw new Error('Screenshot too small') + }) + + await test('getVisibleText()', async () => { + const text = await browser.getVisibleText() + if (!text || text.length < 10) throw new Error('No visible text') + }) + + await test('getVisibleText(selector)', async () => { + await browser.navigate(TEST_URL) + const text = await browser.getVisibleText('body') + if (!text) throw new Error('No text from selector') + }) + + await test('getVisibleHtml()', async () => { + const html = await browser.getVisibleHtml() + if (!html.includes('<')) throw new Error('Invalid HTML') + }) + + await test('getVisibleHtml({ removeScripts: true })', async () => { + const html = await browser.getVisibleHtml({ removeScripts: true, minify: true }) + if (html.includes(' { + const tree = await browser.getAccessibilityTree() + if (!tree) throw new Error('No accessibility tree') + }) + + // Skip PDF - requires non-headless or specific setup + skip('savePdf()', 'Requires specific browser configuration') + + // ============================================ + // INTERACTION + // ============================================ + console.log('\n--- INTERACTION ---') + + await test('navigate to form', async () => { + await browser.navigate(FORM_URL) + await browser.waitForSelector('form') + }) + + await test('fill()', async () => { + await browser.fill('input[name="custname"]', 'Test User') + }) + + await test('type()', async () => { + await browser.fill('input[name="custtel"]', '') // Clear first + await browser.type('input[name="custtel"]', '555-1234', 10) + }) + + await test('click()', async () => { + await browser.click('input[name="topping"][value="bacon"]') + }) + + await test('hover()', async () => { + // Use a reliable element - the form itself or legend + await browser.hover('form') + }) + + await test('select()', async () => { + // httpbin form uses select[name="size"] - use with waitForSelector first + try { + await browser.waitForSelector('select[name="size"]', { timeout: 2000 }) + await browser.select('select[name="size"]', 'medium') + } catch { + // If select doesn't exist, try radio button selection as alternative + await browser.click('input[name="size"][value="medium"]') + } + }) + + await test('pressKey()', async () => { + await browser.pressKey('Tab') + }) + + // Skip drag - requires specific elements + skip('drag()', 'Requires draggable elements') + + // Skip uploadFile - requires file input + skip('uploadFile()', 'Requires file input element') + + // ============================================ + // VIEWPORT + // ============================================ + console.log('\n--- VIEWPORT ---') + + await test('resize()', async () => { + await browser.resize(1920, 1080) + }) + + await test('setDevice()', async () => { + await browser.setDevice('iPhone 12') + }) + + // Reset viewport + await browser.resize(1280, 720) + + // ============================================ + // IFRAME + // ============================================ + console.log('\n--- IFRAME ---') + skip('iframeClick()', 'Requires page with iframe') + skip('iframeFill()', 'Requires page with iframe') + + // ============================================ + // JAVASCRIPT & CONSOLE + // ============================================ + console.log('\n--- JAVASCRIPT & CONSOLE ---') + + await test('evaluate()', async () => { + const result = await browser.evaluate('1 + 1') + if (result !== 2) throw new Error(`Expected 2, got ${result}`) + }) + + await test('evaluate() - DOM access', async () => { + // Use document.body which always exists + const hasBody = await browser.evaluate(() => !!document.body) + if (!hasBody) throw new Error('Could not access DOM via evaluate') + }) + + await test('console logging + getConsoleLogs()', async () => { + await browser.evaluate(() => { + console.log('Test log message') + console.warn('Test warning') + console.error('Test error') + }) + await browser.wait(100) // Let logs propagate + const logs = browser.getConsoleLogs() + if (logs.length < 3) throw new Error(`Expected 3+ logs, got ${logs.length}`) + }) + + await test('getConsoleLogs({ type: "error" })', async () => { + const errors = browser.getConsoleLogs({ type: 'error' }) + if (errors.length < 1) throw new Error('Expected error logs') + }) + + await test('getConsoleLogs({ search: "warning" })', async () => { + const warns = browser.getConsoleLogs({ search: 'warning' }) + if (warns.length < 1) throw new Error('Expected warning logs') + }) + + await test('setUserAgent()', async () => { + await browser.setUserAgent('TestBot/1.0') + // Navigate to check + await browser.navigate('https://httpbin.org/user-agent') + const text = await browser.getVisibleText() + if (!text.includes('TestBot')) throw new Error('User agent not set') + }) + + // ============================================ + // NETWORK MONITORING + // ============================================ + console.log('\n--- NETWORK MONITORING ---') + + await test('clearNetworkLogs()', async () => { + browser.clearNetworkLogs() + }) + + await test('network capture on navigate', async () => { + await browser.navigate('https://httpbin.org/get') + await browser.wait(500) + const logs = browser.getNetworkLogs() + if (logs.length < 1) throw new Error('No network logs captured') + }) + + await test('getNetworkLogs({ type: "response" })', async () => { + const responses = browser.getNetworkLogs({ type: 'response' }) + if (responses.length < 1) throw new Error('No response logs') + }) + + await test('getNetworkLogs({ urlPattern: /get/ })', async () => { + const filtered = browser.getNetworkLogs({ urlPattern: /get/ }) + if (filtered.length < 1) throw new Error('URL pattern filter failed') + }) + + await test('getNetworkLogs({ status: 200 })', async () => { + const ok = browser.getNetworkLogs({ status: 200, type: 'response' }) + if (ok.length < 1) throw new Error('Status filter failed') + }) + + await test('getNetworkStats()', async () => { + const stats = browser.getNetworkStats() + if (stats.totalRequests < 1) throw new Error('No requests in stats') + if (stats.totalResponses < 1) throw new Error('No responses in stats') + }) + + // ============================================ + // DIALOG HANDLING + // ============================================ + console.log('\n--- DIALOG HANDLING ---') + + await test('setDialogHandler()', async () => { + browser.setDialogHandler(true, 'test response') + }) + + await test('getPendingDialog() - no dialog', async () => { + const dialog = browser.getPendingDialog() + if (dialog !== null) throw new Error('Expected no pending dialog') + }) + + // Dialog interaction requires triggering a dialog - skip for now + skip('handleDialog()', 'Requires triggering a dialog') + + // ============================================ + // TAB MANAGEMENT + // ============================================ + console.log('\n--- TAB MANAGEMENT ---') + + await test('getTabs()', async () => { + const tabs = browser.getTabs() + if (tabs.length < 1) throw new Error('Expected at least 1 tab') + }) + + await test('newTab()', async () => { + const beforeCount = browser.getTabs().length + await browser.newTab('https://httpbin.org/html') + const afterCount = browser.getTabs().length + if (afterCount !== beforeCount + 1) throw new Error('Tab not created') + }) + + await test('switchTab()', async () => { + await browser.switchTab(0) + const url = browser.getUrl() + // Should be back on first tab + }) + + await test('closeTab()', async () => { + await browser.switchTab(1) // Switch to second tab + await browser.closeTab() + const tabs = browser.getTabs() + if (tabs.length !== 1) throw new Error('Tab not closed') + }) + + // ============================================ + // WAITING + // ============================================ + console.log('\n--- WAITING ---') + + await test('wait()', async () => { + const start = Date.now() + await browser.wait(100) + const elapsed = Date.now() - start + if (elapsed < 90) throw new Error('Wait too short') + }) + + await test('waitForSelector()', async () => { + await browser.navigate('https://httpbin.org/html') + await browser.waitForSelector('h1') + }) + + await test('waitForText()', async () => { + await browser.waitForText('Herman Melville') + }) + + await test('waitForNavigation()', async () => { + // Already navigated, this should pass immediately or timeout + await browser.navigate('https://httpbin.org') + }) + + await test('waitForNetworkIdle()', async () => { + await browser.waitForNetworkIdle(5000) + }) + + await test('waitForResponse()', async () => { + // Make a request and wait for it + browser.clearNetworkLogs() + const responsePromise = browser.waitForResponse(/httpbin/, { timeout: 5000 }) + await browser.navigate('https://httpbin.org/get') + const response = await responsePromise + if (response.status !== 200) throw new Error(`Expected 200, got ${response.status}`) + }) + + // ============================================ + // CLEANUP + // ============================================ + console.log('\n--- CLEANUP ---') + + await test('close()', async () => { + await browser.close() + }) + + // ============================================ + // SUMMARY + // ============================================ + console.log('\n' + '='.repeat(50)) + console.log('TEST SUMMARY') + console.log('='.repeat(50)) + + const passed = results.filter(r => r.status === 'PASS').length + const failed = results.filter(r => r.status === 'FAIL').length + const skipped = results.filter(r => r.status === 'SKIP').length + + console.log(`\n✅ Passed: ${passed}`) + console.log(`❌ Failed: ${failed}`) + console.log(`⏭️ Skipped: ${skipped}`) + console.log(`📊 Total: ${results.length}`) + + if (failed > 0) { + console.log('\n--- FAILURES ---') + results.filter(r => r.status === 'FAIL').forEach(r => { + console.log(`\n${r.name}:`) + console.log(` Error: ${r.error}`) + }) + } + + const totalDuration = results.reduce((sum, r) => sum + (r.duration || 0), 0) + console.log(`\n⏱️ Total time: ${(totalDuration / 1000).toFixed(2)}s`) + + process.exit(failed > 0 ? 1 : 0) +} + +main().catch(err => { + console.error('Test suite failed:', err) + process.exit(1) +}) diff --git a/.opencode/skills/Utilities/Browser/examples/screenshot.ts b/.opencode/skills/Utilities/Browser/examples/screenshot.ts new file mode 100755 index 00000000..c4aec0e9 --- /dev/null +++ b/.opencode/skills/Utilities/Browser/examples/screenshot.ts @@ -0,0 +1,71 @@ +#!/usr/bin/env bun + +/** + * Take a screenshot of a URL + * + * Usage: bun screenshot.ts [output.png] + * + * Examples: + * bun screenshot.ts https://example.com + * bun screenshot.ts https://example.com page.png + * bun screenshot.ts https://example.com fullpage.png --full + */ + +import { PlaywrightBrowser } from '../index' + +async function main() { + const url = process.argv[2] + const output = process.argv[3] || 'screenshot.png' + const fullPage = process.argv.includes('--full') + + if (!url) { + console.error('Usage: bun screenshot.ts [output.png] [--full]') + console.error('') + console.error('Examples:') + console.error(' bun screenshot.ts https://example.com') + console.error(' bun screenshot.ts https://example.com page.png') + console.error(' bun screenshot.ts https://example.com fullpage.png --full') + process.exit(1) + } + + console.log('=== Playwright Screenshot ===\n') + + const browser = new PlaywrightBrowser() + + try { + console.log('1. Launching browser...') + await browser.launch({ headless: true }) + + console.log(`2. Navigating to ${url}...`) + await browser.navigate(url) + + console.log(`3. Taking screenshot (fullPage: ${fullPage})...`) + await browser.screenshot({ + path: output, + fullPage + }) + + console.log(`\n✅ Screenshot saved: ${output}`) + + // Token savings calculation + const mcpTokens = 13700 + const codeTokens = 150 + const savings = ((mcpTokens - codeTokens) / mcpTokens * 100).toFixed(1) + + console.log(`\n📊 Token savings: ${savings}%`) + console.log(` MCP approach: ~${mcpTokens} tokens`) + console.log(` Code approach: ~${codeTokens} tokens`) + + } catch (error) { + console.error('❌ Error:', error instanceof Error ? error.message : error) + process.exit(1) + } finally { + await browser.close() + } +} + +if (import.meta.main) { + main() +} + +export { main } diff --git a/.opencode/skills/Utilities/Browser/examples/verify-page.ts b/.opencode/skills/Utilities/Browser/examples/verify-page.ts new file mode 100755 index 00000000..737efbd2 --- /dev/null +++ b/.opencode/skills/Utilities/Browser/examples/verify-page.ts @@ -0,0 +1,91 @@ +#!/usr/bin/env bun + +/** + * Verify a page loads correctly + * + * Usage: bun verify-page.ts [selector] + * + * Examples: + * bun verify-page.ts https://example.com + * bun verify-page.ts https://example.com "h1" + * bun verify-page.ts https://example.com ".main-content" + */ + +import { PlaywrightBrowser } from '../index' + +async function main() { + const url = process.argv[2] + const selector = process.argv[3] + + if (!url) { + console.error('Usage: bun verify-page.ts [selector]') + console.error('') + console.error('Examples:') + console.error(' bun verify-page.ts https://example.com') + console.error(' bun verify-page.ts https://example.com "h1"') + process.exit(1) + } + + console.log('=== Page Verification ===\n') + + const browser = new PlaywrightBrowser() + + try { + console.log('1. Launching browser...') + await browser.launch({ headless: true }) + + console.log(`2. Navigating to ${url}...`) + const startTime = Date.now() + await browser.navigate(url) + const loadTime = Date.now() - startTime + + console.log(` ✅ Page loaded in ${loadTime}ms`) + + // Get page title + const title = await browser.getTitle() + console.log(`\n3. Page title: "${title}"`) + + // Get current URL (check for redirects) + const finalUrl = browser.getUrl() + if (finalUrl !== url) { + console.log(` ⚠️ Redirected to: ${finalUrl}`) + } + + // Check for specific selector if provided + if (selector) { + console.log(`\n4. Checking for selector: ${selector}`) + try { + await browser.waitForSelector(selector, { timeout: 5000 }) + const text = await browser.getVisibleText(selector) + console.log(` ✅ Found! Content: "${text.slice(0, 100)}${text.length > 100 ? '...' : ''}"`) + } catch { + console.log(` ❌ Selector not found within 5s`) + } + } + + // Check console for errors + const errors = browser.getConsoleLogs({ type: 'error' }) + if (errors.length > 0) { + console.log(`\n⚠️ Console errors (${errors.length}):`) + errors.slice(0, 3).forEach(err => { + console.log(` - ${err.text.slice(0, 100)}`) + }) + } else { + console.log('\n✅ No console errors') + } + + console.log('\n=== Verification Complete ===') + + } catch (error) { + console.error('❌ Error:', error instanceof Error ? error.message : error) + process.exit(1) + } finally { + await browser.close() + } +} + +if (import.meta.main) { + main() +} + +export { main } diff --git a/.opencode/skills/Utilities/Browser/index.ts b/.opencode/skills/Utilities/Browser/index.ts new file mode 100755 index 00000000..5d06b90f --- /dev/null +++ b/.opencode/skills/Utilities/Browser/index.ts @@ -0,0 +1,907 @@ +/** + * Playwright Code-First Interface + * + * Replaces token-heavy Playwright MCP with direct code execution. + * Savings: ~13,700 tokens (MCP) → ~50-200 tokens (per-operation) + * + * @example + * const browser = new PlaywrightBrowser() + * await browser.launch() + * await browser.navigate('https://example.com') + * const screenshot = await browser.screenshot() + * await browser.close() + */ + +import { chromium, firefox, webkit, type Browser, type Page, type BrowserContext } from 'playwright' + +export type BrowserType = 'chromium' | 'firefox' | 'webkit' + +export interface LaunchOptions { + browser?: BrowserType + headless?: boolean + viewport?: { width: number; height: number } + userAgent?: string +} + +export interface NavigateOptions { + timeout?: number + waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit' +} + +export interface ScreenshotOptions { + selector?: string + fullPage?: boolean + path?: string + type?: 'png' | 'jpeg' + quality?: number +} + +export interface ClickOptions { + button?: 'left' | 'right' | 'middle' + clickCount?: number + delay?: number + timeout?: number +} + +export interface FillOptions { + timeout?: number + force?: boolean +} + +export interface ConsoleLogEntry { + type: string + text: string + timestamp: number +} + +export interface NetworkLogEntry { + type: 'request' | 'response' + url: string + method: string + status?: number + statusText?: string + headers?: Record + resourceType?: string + timestamp: number + duration?: number + size?: number +} + +export interface DialogInfo { + type: 'alert' | 'confirm' | 'prompt' | 'beforeunload' + message: string + defaultValue?: string +} + +/** + * Main Playwright browser controller + * + * Provides all browser automation capabilities without MCP overhead. + * Each method is a direct wrapper around Playwright APIs. + */ +export class PlaywrightBrowser { + private browser: Browser | null = null + private context: BrowserContext | null = null + private page: Page | null = null + private consoleLogs: ConsoleLogEntry[] = [] + private networkLogs: NetworkLogEntry[] = [] + private requestTimings: Map = new Map() + private pendingDialog: DialogInfo | null = null + private autoHandleDialogs: boolean = false + private dialogResponse: string | boolean = true + + /** + * Launch browser instance + */ + async launch(options?: LaunchOptions): Promise { + const browserType = options?.browser || 'chromium' + const launcher = browserType === 'firefox' ? firefox : browserType === 'webkit' ? webkit : chromium + + this.browser = await launcher.launch({ + headless: options?.headless ?? false + }) + + this.context = await this.browser.newContext({ + viewport: options?.viewport || { width: 1280, height: 720 }, + userAgent: options?.userAgent + }) + + this.page = await this.context.newPage() + + // Attach all event listeners + this.attachPageListeners(this.page) + } + + /** + * Ensure browser is launched + */ + private ensurePage(): Page { + if (!this.page) { + throw new Error('Browser not launched. Call launch() first.') + } + return this.page + } + + /** + * Attach event listeners to a page (console, network, dialog) + * Called when creating new pages to maintain consistent monitoring + */ + private attachPageListeners(page: Page): void { + // Capture console logs + page.on('console', msg => { + this.consoleLogs.push({ + type: msg.type(), + text: msg.text(), + timestamp: Date.now() + }) + }) + + // Capture network requests + page.on('request', request => { + const timestamp = Date.now() + this.requestTimings.set(request.url(), timestamp) + this.networkLogs.push({ + type: 'request', + url: request.url(), + method: request.method(), + resourceType: request.resourceType(), + headers: request.headers(), + timestamp + }) + }) + + // Capture network responses + page.on('response', async response => { + const url = response.url() + const requestTime = this.requestTimings.get(url) + const timestamp = Date.now() + + let size = 0 + try { + const body = await response.body() + size = body.length + } catch { + // Response body not available (e.g., redirects) + } + + this.networkLogs.push({ + type: 'response', + url, + method: response.request().method(), + status: response.status(), + statusText: response.statusText(), + headers: response.headers(), + resourceType: response.request().resourceType(), + timestamp, + duration: requestTime ? timestamp - requestTime : undefined, + size + }) + }) + + // Capture dialogs + page.on('dialog', async dialog => { + this.pendingDialog = { + type: dialog.type() as DialogInfo['type'], + message: dialog.message(), + defaultValue: dialog.defaultValue() + } + + if (this.autoHandleDialogs) { + if (typeof this.dialogResponse === 'string') { + await dialog.accept(this.dialogResponse) + } else if (this.dialogResponse) { + await dialog.accept() + } else { + await dialog.dismiss() + } + this.pendingDialog = null + } + }) + } + + // ============================================ + // NAVIGATION + // ============================================ + + /** + * Navigate to URL + */ + async navigate(url: string, options?: NavigateOptions): Promise { + const page = this.ensurePage() + await page.goto(url, { + timeout: options?.timeout || 30000, + waitUntil: options?.waitUntil || 'load' + }) + } + + /** + * Go back in browser history + */ + async goBack(): Promise { + const page = this.ensurePage() + await page.goBack() + } + + /** + * Go forward in browser history + */ + async goForward(): Promise { + const page = this.ensurePage() + await page.goForward() + } + + /** + * Reload current page + */ + async reload(): Promise { + const page = this.ensurePage() + await page.reload() + } + + /** + * Get current URL + */ + getUrl(): string { + return this.ensurePage().url() + } + + /** + * Get page title + */ + async getTitle(): Promise { + return await this.ensurePage().title() + } + + // ============================================ + // CAPTURE + // ============================================ + + /** + * Take screenshot + * + * @returns Base64 encoded image or saves to path + */ + async screenshot(options?: ScreenshotOptions): Promise { + const page = this.ensurePage() + + if (options?.selector) { + const element = await page.locator(options.selector) + return await element.screenshot({ + path: options?.path, + type: options?.type || 'png', + quality: options?.quality + }) + } + + return await page.screenshot({ + path: options?.path, + fullPage: options?.fullPage || false, + type: options?.type || 'png', + quality: options?.quality + }) + } + + /** + * Get visible text content + */ + async getVisibleText(selector?: string): Promise { + const page = this.ensurePage() + + if (selector) { + return await page.locator(selector).textContent() || '' + } + + return await page.evaluate(() => document.body.innerText) + } + + /** + * Get HTML content (with optional cleanup) + */ + async getVisibleHtml(options?: { + selector?: string + removeScripts?: boolean + removeStyles?: boolean + removeComments?: boolean + minify?: boolean + }): Promise { + const page = this.ensurePage() + + let html = options?.selector + ? await page.locator(options.selector).innerHTML() + : await page.content() + + if (options?.removeScripts) { + html = html.replace(/)<[^<]*)*<\/script>/gi, '') + } + + if (options?.removeStyles) { + html = html.replace(/)<[^<]*)*<\/style>/gi, '') + } + + if (options?.removeComments) { + html = html.replace(//g, '') + } + + if (options?.minify) { + html = html.replace(/\s+/g, ' ').trim() + } + + return html + } + + /** + * Save page as PDF + */ + async savePdf(path: string, options?: { + format?: 'A4' | 'Letter' | 'Legal' | 'Tabloid' + printBackground?: boolean + margin?: { top?: string; right?: string; bottom?: string; left?: string } + }): Promise { + const page = this.ensurePage() + + return await page.pdf({ + path, + format: options?.format || 'A4', + printBackground: options?.printBackground ?? true, + margin: options?.margin + }) + } + + // ============================================ + // INTERACTION + // ============================================ + + /** + * Click element + */ + async click(selector: string, options?: ClickOptions): Promise { + const page = this.ensurePage() + await page.click(selector, { + button: options?.button, + clickCount: options?.clickCount, + delay: options?.delay, + timeout: options?.timeout + }) + } + + /** + * Hover over element + */ + async hover(selector: string): Promise { + const page = this.ensurePage() + await page.hover(selector) + } + + /** + * Fill input field + */ + async fill(selector: string, value: string, options?: FillOptions): Promise { + const page = this.ensurePage() + await page.fill(selector, value, { + timeout: options?.timeout, + force: options?.force + }) + } + + /** + * Type text (character by character, for realistic input) + */ + async type(selector: string, text: string, delay?: number): Promise { + const page = this.ensurePage() + await page.locator(selector).pressSequentially(text, { delay: delay || 50 }) + } + + /** + * Select dropdown option + */ + async select(selector: string, value: string | string[]): Promise { + const page = this.ensurePage() + await page.selectOption(selector, value) + } + + /** + * Press keyboard key + */ + async pressKey(key: string, selector?: string): Promise { + const page = this.ensurePage() + + if (selector) { + await page.locator(selector).press(key) + } else { + await page.keyboard.press(key) + } + } + + /** + * Drag element to target + */ + async drag(sourceSelector: string, targetSelector: string): Promise { + const page = this.ensurePage() + await page.dragAndDrop(sourceSelector, targetSelector) + } + + /** + * Upload file + */ + async uploadFile(selector: string, filePath: string | string[]): Promise { + const page = this.ensurePage() + await page.setInputFiles(selector, filePath) + } + + // ============================================ + // VIEWPORT + // ============================================ + + /** + * Resize viewport + */ + async resize(width: number, height: number): Promise { + const page = this.ensurePage() + await page.setViewportSize({ width, height }) + } + + /** + * Set device emulation + */ + async setDevice(device: string): Promise { + const page = this.ensurePage() + const devices = await import('playwright').then(m => m.devices) + const deviceConfig = devices[device] + + if (!deviceConfig) { + throw new Error(`Unknown device: ${device}. See Playwright devices list.`) + } + + await page.setViewportSize(deviceConfig.viewport) + } + + // ============================================ + // IFRAME SUPPORT + // ============================================ + + /** + * Click element inside iframe + */ + async iframeClick(iframeSelector: string, elementSelector: string): Promise { + const page = this.ensurePage() + const frame = page.frameLocator(iframeSelector) + await frame.locator(elementSelector).click() + } + + /** + * Fill input inside iframe + */ + async iframeFill(iframeSelector: string, elementSelector: string, value: string): Promise { + const page = this.ensurePage() + const frame = page.frameLocator(iframeSelector) + await frame.locator(elementSelector).fill(value) + } + + // ============================================ + // JAVASCRIPT EXECUTION + // ============================================ + + /** + * Execute JavaScript in page context + */ + async evaluate(script: string | (() => T)): Promise { + const page = this.ensurePage() + return await page.evaluate(script as any) + } + + /** + * Get console logs + */ + getConsoleLogs(options?: { + type?: 'all' | 'error' | 'warning' | 'log' | 'info' | 'debug' + search?: string + limit?: number + clear?: boolean + }): ConsoleLogEntry[] { + let logs = [...this.consoleLogs] + + if (options?.type && options.type !== 'all') { + logs = logs.filter(log => log.type === options.type) + } + + if (options?.search) { + logs = logs.filter(log => log.text.includes(options.search!)) + } + + if (options?.limit) { + logs = logs.slice(-options.limit) + } + + if (options?.clear) { + this.consoleLogs = [] + } + + return logs + } + + /** + * Set custom user agent + */ + async setUserAgent(userAgent: string): Promise { + if (!this.context) { + throw new Error('Browser not launched. Call launch() first.') + } + + // Need to create new page with new context for UA change + const newContext = await this.browser!.newContext({ userAgent }) + const newPage = await newContext.newPage() + + // Attach event listeners to new page + this.attachPageListeners(newPage) + + await this.context.close() + this.context = newContext + this.page = newPage + } + + // ============================================ + // NETWORK MONITORING (matches browser_network_requests) + // ============================================ + + /** + * Get network logs (requests and responses) + * + * @example + * const logs = browser.getNetworkLogs({ type: 'response', status: 200 }) + * const apiCalls = browser.getNetworkLogs({ urlPattern: /api/ }) + * const errors = browser.getNetworkLogs({ status: [400, 401, 403, 404, 500] }) + */ + getNetworkLogs(options?: { + type?: 'request' | 'response' | 'all' + urlPattern?: string | RegExp + method?: string | string[] + status?: number | number[] + resourceType?: string | string[] + limit?: number + clear?: boolean + }): NetworkLogEntry[] { + let logs = [...this.networkLogs] + + // Filter by type + if (options?.type && options.type !== 'all') { + logs = logs.filter(log => log.type === options.type) + } + + // Filter by URL pattern + if (options?.urlPattern) { + const pattern = options.urlPattern instanceof RegExp + ? options.urlPattern + : new RegExp(options.urlPattern) + logs = logs.filter(log => pattern.test(log.url)) + } + + // Filter by method + if (options?.method) { + const methods = Array.isArray(options.method) ? options.method : [options.method] + logs = logs.filter(log => methods.includes(log.method)) + } + + // Filter by status (responses only) + if (options?.status) { + const statuses = Array.isArray(options.status) ? options.status : [options.status] + logs = logs.filter(log => log.status && statuses.includes(log.status)) + } + + // Filter by resource type + if (options?.resourceType) { + const types = Array.isArray(options.resourceType) ? options.resourceType : [options.resourceType] + logs = logs.filter(log => log.resourceType && types.includes(log.resourceType)) + } + + // Limit results + if (options?.limit) { + logs = logs.slice(-options.limit) + } + + // Clear logs if requested + if (options?.clear) { + this.networkLogs = [] + this.requestTimings.clear() + } + + return logs + } + + /** + * Clear all network logs + */ + clearNetworkLogs(): void { + this.networkLogs = [] + this.requestTimings.clear() + } + + /** + * Get network summary statistics + */ + getNetworkStats(): { + totalRequests: number + totalResponses: number + byStatus: Record + byResourceType: Record + totalSize: number + avgDuration: number + } { + const responses = this.networkLogs.filter(l => l.type === 'response') + + const byStatus: Record = {} + const byResourceType: Record = {} + let totalSize = 0 + let totalDuration = 0 + let durationCount = 0 + + for (const log of responses) { + if (log.status) { + byStatus[log.status] = (byStatus[log.status] || 0) + 1 + } + if (log.resourceType) { + byResourceType[log.resourceType] = (byResourceType[log.resourceType] || 0) + 1 + } + totalSize += log.size || 0 + if (log.duration) { + totalDuration += log.duration + durationCount++ + } + } + + return { + totalRequests: this.networkLogs.filter(l => l.type === 'request').length, + totalResponses: responses.length, + byStatus, + byResourceType, + totalSize, + avgDuration: durationCount > 0 ? totalDuration / durationCount : 0 + } + } + + // ============================================ + // DIALOG HANDLING (matches browser_handle_dialog) + // ============================================ + + /** + * Configure automatic dialog handling + * + * @param auto - Whether to auto-handle dialogs + * @param response - Response for prompts (string) or confirm/alert (boolean) + */ + setDialogHandler(auto: boolean, response?: string | boolean): void { + this.autoHandleDialogs = auto + this.dialogResponse = response ?? true + } + + /** + * Get pending dialog (if any) + */ + getPendingDialog(): DialogInfo | null { + return this.pendingDialog + } + + /** + * Handle pending dialog manually + */ + async handleDialog(action: 'accept' | 'dismiss', promptText?: string): Promise { + const page = this.ensurePage() + + // Wait briefly for dialog if not already captured + if (!this.pendingDialog) { + await this.wait(100) + } + + if (!this.pendingDialog) { + throw new Error('No pending dialog to handle') + } + + // The dialog was already stored, we need to wait for the next one if already handled + // This is a simplification - for more complex cases, we'd queue dialogs + page.once('dialog', async dialog => { + if (action === 'accept') { + await dialog.accept(promptText) + } else { + await dialog.dismiss() + } + }) + + this.pendingDialog = null + } + + // ============================================ + // WAIT FOR TEXT (matches browser_wait_for) + // ============================================ + + /** + * Wait for text to appear or disappear + */ + async waitForText(text: string, options?: { + state?: 'visible' | 'hidden' + timeout?: number + }): Promise { + const page = this.ensurePage() + const locator = page.getByText(text) + + if (options?.state === 'hidden') { + await locator.waitFor({ state: 'hidden', timeout: options?.timeout }) + } else { + await locator.waitFor({ state: 'visible', timeout: options?.timeout }) + } + } + + // ============================================ + // TAB MANAGEMENT (matches browser_tabs) + // ============================================ + + /** + * Get all open tabs/pages + */ + getTabs(): { url: string; title: string; index: number }[] { + if (!this.context) { + throw new Error('Browser not launched. Call launch() first.') + } + + return this.context.pages().map((page, index) => ({ + url: page.url(), + title: '', // Would need async to get title + index + })) + } + + /** + * Create new tab + */ + async newTab(url?: string): Promise { + if (!this.context) { + throw new Error('Browser not launched. Call launch() first.') + } + + this.page = await this.context.newPage() + + // Attach all event listeners to new page + this.attachPageListeners(this.page) + + if (url) { + await this.page.goto(url) + } + } + + /** + * Switch to tab by index + */ + async switchTab(index: number): Promise { + if (!this.context) { + throw new Error('Browser not launched. Call launch() first.') + } + + const pages = this.context.pages() + if (index < 0 || index >= pages.length) { + throw new Error(`Tab index ${index} out of range (0-${pages.length - 1})`) + } + + this.page = pages[index] + await this.page.bringToFront() + } + + /** + * Close current tab + */ + async closeTab(): Promise { + const page = this.ensurePage() + await page.close() + + // Switch to another tab if available + if (this.context) { + const pages = this.context.pages() + this.page = pages.length > 0 ? pages[pages.length - 1] : null + } + } + + // ============================================ + // WAITING + // ============================================ + + /** + * Wait for element + */ + async waitForSelector(selector: string, options?: { + state?: 'attached' | 'detached' | 'visible' | 'hidden' + timeout?: number + }): Promise { + const page = this.ensurePage() + await page.waitForSelector(selector, { + state: options?.state, + timeout: options?.timeout + }) + } + + /** + * Wait for navigation + */ + async waitForNavigation(options?: { + url?: string | RegExp + timeout?: number + }): Promise { + const page = this.ensurePage() + await page.waitForURL(options?.url || '**/*', { + timeout: options?.timeout + }) + } + + /** + * Wait for network idle + */ + async waitForNetworkIdle(timeout?: number): Promise { + const page = this.ensurePage() + await page.waitForLoadState('networkidle', { timeout }) + } + + /** + * Wait fixed time (use sparingly) + */ + async wait(ms: number): Promise { + await new Promise(resolve => setTimeout(resolve, ms)) + } + + // ============================================ + // RESPONSE MONITORING + // ============================================ + + /** + * Wait for specific response + */ + async waitForResponse(urlPattern: string | RegExp, options?: { + timeout?: number + }): Promise<{ status: number; body: string }> { + const page = this.ensurePage() + + const response = await page.waitForResponse(urlPattern, { + timeout: options?.timeout + }) + + return { + status: response.status(), + body: await response.text() + } + } + + // ============================================ + // ACCESSIBILITY + // ============================================ + + /** + * Get accessibility tree snapshot + * Uses ARIA snapshot for accessibility-focused content representation + */ + async getAccessibilityTree(): Promise { + const page = this.ensurePage() + // ariaSnapshot provides accessibility tree representation + return await page.locator(':root').ariaSnapshot() + } + + // ============================================ + // CLEANUP + // ============================================ + + /** + * Close browser + */ + async close(): Promise { + if (this.browser) { + await this.browser.close() + this.browser = null + this.context = null + this.page = null + this.consoleLogs = [] + this.networkLogs = [] + this.requestTimings.clear() + this.pendingDialog = null + } + } +} + +// Export singleton for simple usage +export const browser = new PlaywrightBrowser() + +// Export types (using type-only export to avoid runtime issues) +export type { Browser, Page, BrowserContext } diff --git a/.opencode/skills/Utilities/Browser/package.json b/.opencode/skills/Utilities/Browser/package.json new file mode 100755 index 00000000..a2935737 --- /dev/null +++ b/.opencode/skills/Utilities/Browser/package.json @@ -0,0 +1,16 @@ +{ + "name": "@kai/playwright", + "version": "2.0.0", + "description": "Code-first Playwright wrapper replacing token-heavy MCP", + "type": "module", + "main": "index.ts", + "scripts": { + "test": "bun test" + }, + "dependencies": { + "playwright": "^1.49.0" + }, + "devDependencies": { + "@types/bun": "^1.1.0" + } +} diff --git a/.opencode/skills/Utilities/Browser/tsconfig.json b/.opencode/skills/Utilities/Browser/tsconfig.json new file mode 100755 index 00000000..7397b8dc --- /dev/null +++ b/.opencode/skills/Utilities/Browser/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "outDir": "./dist", + "declaration": true, + "types": ["bun-types"] + }, + "include": ["*.ts", "tools/*.ts", "examples/*.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/.opencode/skills/Utilities/Cloudflare/SKILL.md b/.opencode/skills/Utilities/Cloudflare/SKILL.md new file mode 100644 index 00000000..9f510d28 --- /dev/null +++ b/.opencode/skills/Utilities/Cloudflare/SKILL.md @@ -0,0 +1,105 @@ +--- +name: Cloudflare +description: Deploy Cloudflare Workers/Pages. USE WHEN Cloudflare, worker, deploy, Pages, MCP server. SkillSearch('cloudflare') for docs. +--- + +## Customization + +**Before executing, check for user customizations at:** +`~/.opencode/skills/PAI/USER/SKILLCUSTOMIZATIONS/Cloudflare/` + +If this directory exists, load and apply any PREFERENCES.md, configurations, or resources found there. These override default behavior. If the directory does not exist, proceed with skill defaults. + + +## 🚨 MANDATORY: Voice Notification (REQUIRED BEFORE ANY ACTION) + +**You MUST send this notification BEFORE doing anything else when this skill is invoked.** + +1. **Send voice notification**: + ```bash + curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the WORKFLOWNAME workflow in the Cloudflare skill to ACTION"}' \ + > /dev/null 2>&1 & + ``` + +2. **Output text notification**: + ``` + Running the **WorkflowName** workflow in the **Cloudflare** skill to ACTION... + ``` + +**This is not optional. Execute this curl command immediately upon skill invocation.** + +# Cloudflare Skill + +Deploy and manage Cloudflare Workers, MCP servers, and Pages. + + +## Workflow Routing + +**When executing a workflow, output this notification directly:** + +``` +Running the **WorkflowName** workflow in the **Cloudflare** skill to ACTION... +``` + + - **Create** MCP server or Worker → `Workflows/Create.md` + - **Troubleshoot** deployment issues → `Workflows/Troubleshoot.md` + +## Quick Reference + +- **Account ID:** Set via `CF_ACCOUNT_ID` environment variable +- **Worker URL format:** `https://[worker-name].[your-subdomain].workers.dev` + +## Deployment Commands + +### Workers Deployment +```bash +# Unset tokens that interfere with wrangler login-based auth +(unset CF_API_TOKEN && unset CLOUDFLARE_API_TOKEN && wrangler deploy) +``` + +### Pages Deployment + +**🚨 CRITICAL: ALL env tokens lack Pages permissions. MUST unset them to use OAuth:** + +```bash +# ALWAYS unset tokens for Pages - OAuth login works, tokens don't +(unset CF_API_TOKEN && unset CLOUDFLARE_API_TOKEN && npx wrangler pages deploy dist --project-name=PROJECT_NAME --commit-dirty=true) +``` + +**Known Pages Projects:** +| Project | Directory | Deploy Command | +|---------|-----------|----------------| +| [project] | `~/Projects/[project]` | `(unset CF_API_TOKEN && unset CLOUDFLARE_API_TOKEN && npx wrangler pages deploy dist --project-name=[project] --commit-dirty=true)` | + +## Critical Notes + +- **Workers:** Unset `CF_API_TOKEN` and `CLOUDFLARE_API_TOKEN` before deploying - they interfere with wrangler login-based auth +- **Pages:** 🚨 **UNSET ALL TOKENS** - None of the API tokens have Pages permissions. OAuth-based wrangler login is the ONLY method that works. + +## Examples + +**Example 1: Deploy a Worker** +``` +User: "deploy the MCP server to Cloudflare" +→ Invokes CREATE workflow +→ Unsets env tokens, runs wrangler deploy +→ "Deployed to https://mcp-server.[subdomain].workers.dev" +``` + +**Example 2: Deploy Pages site** +``` +User: "deploy my-app to Cloudflare" +→ Builds dist/, unsets tokens +→ Runs wrangler pages deploy +→ "Deployed my-app to Cloudflare Pages" +``` + +**Example 3: Fix deployment error** +``` +User: "Cloudflare deploy is failing with auth error" +→ Invokes TROUBLESHOOT workflow +→ Identifies token interference +→ "Fixed - tokens were overriding OAuth. Redeployed successfully." +``` diff --git a/.opencode/skills/Utilities/Cloudflare/Workflows/Create.md b/.opencode/skills/Utilities/Cloudflare/Workflows/Create.md new file mode 100644 index 00000000..3e5107ee --- /dev/null +++ b/.opencode/skills/Utilities/Cloudflare/Workflows/Create.md @@ -0,0 +1,76 @@ +# Create Cloudflare Worker or MCP Server + +Deploy a new Cloudflare Worker or MCP server. + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the Create workflow in the Cloudflare skill to deploy a new worker"}' \ + > /dev/null 2>&1 & +``` + +Running the **Create** workflow in the **Cloudflare** skill to deploy a new worker... + +## Project Structure + +``` +workers/mcp-server-name/ +├── src/ +│ └── simple.js # Main worker code +├── wrangler.toml # Cloudflare config +├── package.json # Dependencies +└── README.md # Documentation +``` + +## Essential wrangler.toml + +```toml +name = "mcp-server-name" +main = "src/simple.js" +compatibility_date = "2024-01-01" +account_id = "$CLOUDFLARE_ACCOUNT_ID" + +[vars] +MCP_SERVER_NAME = "your-server" +MCP_SERVER_VERSION = "1.0.0" +``` + +## Deployment + +```bash +# CRITICAL: Must unset environment variables that interfere +cd workers/mcp-server-name +(unset CF_API_TOKEN && unset CLOUDFLARE_API_TOKEN && wrangler deploy) +``` + +## MCP Server Endpoints + +- Root `/` - Server info +- `/tools` - List available tools +- `/call` - Execute tools (POST) +- Always include CORS headers +- Return proper JSON responses + +## Testing + +```bash +# Test root endpoint +curl https://your-server.your-account.workers.dev/ + +# Test tool execution +curl -X POST https://your-server.your-account.workers.dev/call \ + -H "Content-Type: application/json" \ + -d '{"name": "tool_name", "arguments": {...}}' +``` + +## Key Details + +- **Account ID:** Configure your Cloudflare account ID in wrangler.toml +- **Always unset** CF_API_TOKEN and CLOUDFLARE_API_TOKEN before deploying +- **URL format:** `https://[worker-name].your-account.workers.dev` + +## Cloudflare Documentation + +For MCP server specifics: https://developers.cloudflare.com/agents/guides/remote-mcp-server/ diff --git a/.opencode/skills/Utilities/Cloudflare/Workflows/Troubleshoot.md b/.opencode/skills/Utilities/Cloudflare/Workflows/Troubleshoot.md new file mode 100644 index 00000000..2edb7228 --- /dev/null +++ b/.opencode/skills/Utilities/Cloudflare/Workflows/Troubleshoot.md @@ -0,0 +1,555 @@ +# troubleshoot-cloudflare-deployment + +Comprehensive Cloudflare deployment troubleshooting workflow with recursive testing and automated fixes using Chrome MCP tools for visual verification. + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the Troubleshoot workflow in the Cloudflare skill to debug deployment issues"}' \ + > /dev/null 2>&1 & +``` + +Running the **Troubleshoot** workflow in the **Cloudflare** skill to debug deployment issues... + +## When to Use This Command + +**USE THIS COMMAND WHEN:** +- Deploying to Cloudflare +- Website deployment succeeds but site doesn't work +- Wrangler deployment errors occur +- Need to verify deployment is actually working +- Site shows errors after deployment +- Need to check logs from failed deployments + +## What This Command Does + +This command implements a recursive test-fix-verify loop: + +1. **Check Deployment Status**: Query Cloudflare API for deployment state +2. **Read Deployment Logs**: Fetch and analyze logs from failed deployments +3. **Test with Chrome MCP**: Verify the deployed site actually works +4. **Identify Issues**: Analyze console errors, network failures, visual problems +5. **Apply Fixes**: Make corrections based on identified issues +6. **Re-deploy**: Push fixes to trigger new deployment +7. **Repeat**: Loop until deployment succeeds AND site works correctly + +**🚨 CRITICAL**: This command uses Chrome MCP tools (PRIMARY) to ACTUALLY TEST the deployed site, not just check if the deployment succeeded! + +# NEVER USE FUCKING CURL, ONLY BROWSER-AUTOMATION (or Chrome MCP fallback) + +!!!! ONLY Use Chrome MCP tools (`Chrome MCP tools` then `browser navigate/screenshot/act`) to control Chrome and test whether the application site is loading and what its behavior is + +- Cloudflare API token in .env file: CLOUDFLARE_API_TOKEN_WORKERS_EDIT + +## Implementation + +```typescript +#!/usr/bin/env bun + +/** + * Cloudflare Deployment Troubleshooting Tool + * + * Recursively tests and fixes Cloudflare deployments until they work properly. + * Uses Chrome MCP tools (PRIMARY) for real browser testing to verify deployments. + * Falls back to Chrome MCP if Chrome MCP unavailable. + */ + +import { $ } from "bun"; +import { readFileSync } from "fs"; +import { join } from "path"; + +interface DeploymentInfo { + id: string; + status: "success" | "failure" | "in_progress"; + url: string; + created_at: string; + latest_stage: { + name: string; + status: string; + ended_at: string | null; + }; +} + +interface TestResult { + passed: boolean; + issues: string[]; + consoleErrors: any[]; + networkErrors: any[]; + visualIssues: string[]; +} + +const CLOUDFLARE_ACCOUNT_ID = "your-account-id-here"; +const PROJECT_NAME = "your-project-name"; // Replace with your project +const MAX_RETRIES = 5; +const DEPLOYMENT_WAIT_TIME = 180000; // 3 minutes in ms + +/** + * Load Cloudflare API token from environment + */ +function loadApiToken(): string { + const envPath = join(process.env.HOME!, "Projects/your-project/.env"); + + try { + const envContent = readFileSync(envPath, "utf-8"); + const match = envContent.match(/CF_API_TOKEN=(.+)/); + + if (!match) { + throw new Error("CLOUDFLARE_API_TOKEN_WORKERS_EDIT not found in .env file"); + } + + return match[1].trim(); + } catch (error) { + console.error("❌ Error loading API token:", error); + throw error; + } +} + +/** + * Fetch latest deployment info from Cloudflare API + */ +async function getLatestDeployment(apiToken: string): Promise { + try { + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/pages/projects/${PROJECT_NAME}/deployments`, + { + headers: { + "Authorization": `Bearer ${apiToken}`, + "Content-Type": "application/json", + }, + } + ); + + const data = await response.json(); + + if (!data.success || !data.result || data.result.length === 0) { + console.error("❌ Failed to fetch deployments:", data.errors); + return null; + } + + return data.result[0]; // Latest deployment + } catch (error) { + console.error("❌ Error fetching deployment:", error); + return null; + } +} + +/** + * Fetch deployment logs from Cloudflare API + */ +async function getDeploymentLogs(apiToken: string, deploymentId: string): Promise { + try { + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/pages/projects/${PROJECT_NAME}/deployments/${deploymentId}/history/logs`, + { + headers: { + "Authorization": `Bearer ${apiToken}`, + "Content-Type": "application/json", + }, + } + ); + + const data = await response.json(); + + if (!data.success) { + console.error("❌ Failed to fetch logs:", data.errors); + return "Failed to fetch logs"; + } + + // Format logs for readability + const logs = data.result.data || []; + return logs.map((log: any) => `[${log.timestamp}] ${log.message}`).join("\n"); + } catch (error) { + console.error("❌ Error fetching logs:", error); + return `Error fetching logs: ${error}`; + } +} + +/** + * Test the deployed site using Chrome MCP + * This is where we verify the deployment ACTUALLY WORKS + */ +async function testDeploymentWithChrome(url: string): Promise { + console.log("\n🧪 Testing deployment with Chrome MCP..."); + console.log(` URL: ${url}`); + + const result: TestResult = { + passed: true, + issues: [], + consoleErrors: [], + networkErrors: [], + visualIssues: [], + }; + + try { + // Note: These Chrome MCP commands should be executed by the OpenCode agent + // This TypeScript code indicates what should be done + console.log(" → Navigate to site"); + console.log(" → Take screenshot"); + console.log(" → Check console errors"); + console.log(" → Check network requests"); + + // The agent should use: + // mcp__Chrome__navigate_page({ url }) + // mcp__Chrome__take_screenshot() + // mcp__Chrome__list_console_messages() + // mcp__Chrome__list_network_requests() + + result.issues.push("AGENT_ACTION_REQUIRED: Use Chrome MCP to test deployment"); + + } catch (error) { + result.passed = false; + result.issues.push(`Chrome testing failed: ${error}`); + } + + return result; +} + +/** + * Analyze deployment logs to identify common issues + */ +function analyzeLogs(logs: string): string[] { + const issues: string[] = []; + + // Common error patterns + const patterns = [ + { pattern: /Error: Cannot find module/, issue: "Missing module dependency" }, + { pattern: /ENOENT: no such file/, issue: "Missing file reference" }, + { pattern: /Failed to load resource/, issue: "Resource loading failed" }, + { pattern: /SyntaxError/, issue: "JavaScript syntax error" }, + { pattern: /Build failed/, issue: "Build process failed" }, + { pattern: /memory/, issue: "Memory limit exceeded" }, + { pattern: /timeout/i, issue: "Build timeout" }, + ]; + + for (const { pattern, issue } of patterns) { + if (pattern.test(logs)) { + issues.push(issue); + } + } + + return issues; +} + +/** + * Main troubleshooting workflow + */ +async function main() { + console.log("🔍 Cloudflare Deployment Troubleshooter"); + console.log("========================================\n"); + + const apiToken = loadApiToken(); + let attemptCount = 0; + let deploymentFixed = false; + + while (attemptCount < MAX_RETRIES && !deploymentFixed) { + attemptCount++; + console.log(`\n📋 Attempt ${attemptCount}/${MAX_RETRIES}`); + console.log("─".repeat(50)); + + // Step 1: Get latest deployment + console.log("\n1️⃣ Checking deployment status..."); + const deployment = await getLatestDeployment(apiToken); + + if (!deployment) { + console.log("❌ Could not fetch deployment information"); + process.exit(1); + } + + console.log(` Status: ${deployment.status}`); + console.log(` URL: ${deployment.url}`); + console.log(` Created: ${deployment.created_at}`); + console.log(` Stage: ${deployment.latest_stage.name} (${deployment.latest_stage.status})`); + + // Step 2: If deployment in progress, wait + if (deployment.status === "in_progress") { + console.log(`\n⏳ Deployment in progress, waiting ${DEPLOYMENT_WAIT_TIME / 1000} seconds...`); + await new Promise(resolve => setTimeout(resolve, DEPLOYMENT_WAIT_TIME)); + continue; + } + + // Step 3: If deployment failed, analyze logs + if (deployment.status === "failure") { + console.log("\n2️⃣ Fetching deployment logs..."); + const logs = await getDeploymentLogs(apiToken, deployment.id); + console.log("\n📄 Deployment Logs:"); + console.log("─".repeat(50)); + console.log(logs); + console.log("─".repeat(50)); + + // Analyze logs for issues + const issues = analyzeLogs(logs); + if (issues.length > 0) { + console.log("\n3️⃣ Identified Issues:"); + issues.forEach((issue, i) => { + console.log(` ${i + 1}. ${issue}`); + }); + + console.log("\n🔧 AGENT ACTION REQUIRED:"); + console.log(" → Analyze the logs above"); + console.log(" → Identify the root cause"); + console.log(" → Apply fixes to the codebase"); + console.log(" → Commit and push fixes"); + console.log(" → Re-run this command to verify"); + } else { + console.log("\n⚠️ No obvious issues found in logs"); + console.log(" Manual investigation required"); + } + + process.exit(1); + } + + // Step 4: Deployment succeeded, now TEST with Chrome MCP + console.log("\n4️⃣ Deployment succeeded! Now testing with Chrome MCP..."); + + const testResult = await testDeploymentWithChrome(deployment.url); + + if (!testResult.passed) { + console.log("\n❌ Deployment succeeded but site has issues!"); + console.log("\n🔍 Issues Found:"); + testResult.issues.forEach((issue, i) => { + console.log(` ${i + 1}. ${issue}`); + }); + + if (testResult.consoleErrors.length > 0) { + console.log("\n📋 Console Errors:"); + testResult.consoleErrors.forEach(err => { + console.log(` → ${err.message}`); + }); + } + + if (testResult.networkErrors.length > 0) { + console.log("\n🌐 Network Errors:"); + testResult.networkErrors.forEach(err => { + console.log(` → ${err.url} (${err.status})`); + }); + } + + if (testResult.visualIssues.length > 0) { + console.log("\n👁️ Visual Issues:"); + testResult.visualIssues.forEach(issue => { + console.log(` → ${issue}`); + }); + } + + console.log("\n🔧 AGENT ACTION REQUIRED:"); + console.log(" → Review issues above"); + console.log(" → Fix problems in codebase"); + console.log(" → Test locally first!"); + console.log(" → Commit and push fixes"); + console.log(" → Re-run this command to verify"); + + process.exit(1); + } + + // Success! + console.log("\n✅ Deployment successful and site is working!"); + console.log(` 🌐 Live at: ${deployment.url}`); + deploymentFixed = true; + } + + if (!deploymentFixed) { + console.log(`\n❌ Failed to fix deployment after ${MAX_RETRIES} attempts`); + console.log(" Manual intervention required"); + process.exit(1); + } +} + +// Run main function +main().catch(error => { + console.error("Fatal error:", error); + process.exit(1); +}); +``` + +## Usage + +```bash +# Run the troubleshooting workflow +troubleshoot-cloudflare-deployment + +# The command will: +# 1. Check deployment status +# 2. Show logs if failed +# 3. Test with Chrome MCP if succeeded +# 4. Report all issues found +# 5. Wait for you to fix and re-run +``` + +## Workflow Steps + +### 1. Check Deployment Status +- Queries Cloudflare API for latest deployment +- Shows status, URL, and stage information +- Waits if deployment is in progress + +### 2. Analyze Logs (if failed) +- Fetches full deployment logs via API +- Scans for common error patterns: + - Missing modules + - File not found errors + - Build failures + - Memory issues + - Timeouts +- Displays identified issues + +### 3. Test with Chrome MCP (if succeeded) +**🚨 CRITICAL**: This is where we verify deployment ACTUALLY WORKS! + +The agent should use Chrome MCP to: +- Navigate to the deployment URL +- Take screenshot for visual confirmation +- Check console for JavaScript errors +- Monitor network requests for failures +- Verify page loads completely +- Check for visual rendering issues + +### 4. Report Issues +- Lists all problems found +- Categorizes by type (console, network, visual) +- Provides specific error messages +- Suggests next steps + +### 5. Wait for Fixes +- Pauses for you to fix issues +- Instructions to test locally first +- Reminds to commit and push +- Ready to re-test after fixes + +### 6. Recursive Loop +- Automatically re-checks after fixes +- Continues until deployment works correctly +- Maximum 5 attempts to prevent infinite loops + +## Chrome MCP Testing Integration + +When the command reaches the testing phase, the OpenCode agent should execute these Chrome MCP commands: + +```javascript +// Navigate to deployment URL +await mcp__Chrome__navigate_page({ + url: deployment.url +}); + +// Take screenshot for visual verification +await mcp__Chrome__take_snapshot(); + +// Check console for errors +const consoleMsgs = await mcp__Chrome__list_console_messages(); +const errors = consoleMsgs.filter(msg => msg.level === 'error'); + +// Check network requests +const networkReqs = await mcp__Chrome__list_network_requests({ + resourceTypes: ['document', 'stylesheet', 'script', 'xhr', 'fetch'] +}); +const failedReqs = networkReqs.filter(req => req.status >= 400); + +// Report findings +if (errors.length > 0 || failedReqs.length > 0) { + console.log("❌ Issues found during Chrome testing!"); + // List specific issues... +} +``` + +## Common Issues and Fixes + +### Build Failures +**Issue**: Build process fails during deployment +**Logs**: "Build failed" or "Command failed with exit code 1" +**Fix**: +- Run `bun run build` locally to see errors +- Fix compilation/build errors +- Commit and push + +### Missing Dependencies +**Issue**: Module not found errors +**Logs**: "Cannot find module 'package-name'" +**Fix**: +- Add missing dependencies to package.json +- Run `bun install` to verify +- Commit package.json and bun.lockb + +### Resource Loading Failures +**Issue**: Site loads but resources 404 +**Chrome**: Network tab shows failed requests +**Fix**: +- Check file paths in code +- Ensure files are committed to git +- Verify _redirects file + +### JavaScript Errors +**Issue**: Console shows runtime errors +**Chrome**: Console errors visible +**Fix**: +- Fix JavaScript errors locally +- Test thoroughly before pushing +- Check browser compatibility + +### Memory/Timeout Issues +**Issue**: Build exceeds limits +**Logs**: "memory" or "timeout" messages +**Fix**: +- Reduce build size +- Optimize dependencies +- Split large bundles + +## Example Output + +``` +🔍 Cloudflare Deployment Troubleshooter +======================================== + +📋 Attempt 1/5 +────────────────────────────────────────────────── + +1️⃣ Checking deployment status... + Status: failure + URL: https://abc123.website.pages.dev + Created: 2025-01-10T14:30:00Z + Stage: build_and_deploy (failed) + +2️⃣ Fetching deployment logs... + +📄 Deployment Logs: +────────────────────────────────────────────────── +[14:30:15] Installing dependencies... +[14:30:20] Running build command: bun run build +[14:30:25] Error: Cannot find module '@vue/shared' +[14:30:25] Build failed with exit code 1 +────────────────────────────────────────────────── + +3️⃣ Identified Issues: + 1. Missing module dependency + 2. Build process failed + +🔧 AGENT ACTION REQUIRED: + → Analyze the logs above + → Identify the root cause + → Apply fixes to the codebase + → Commit and push fixes + → Re-run this command to verify +``` + +## Integration with Other Commands + +This command works well with: +- `publish-blog` - Automatically called after blog publishing +- `write-blog` - Use before publishing to catch issues early +- `browser-tools-setup` - Ensure Chrome MCP is ready for testing + +## Best Practices + +1. **Always test locally first** - Run `bun run build` before pushing +2. **Use Chrome MCP for verification** - Don't trust deployment status alone +3. **Check all error types** - Console, network, and visual issues +4. **Fix one issue at a time** - Don't try to fix everything at once +5. **Re-test after each fix** - Ensure the fix actually worked + +## Notes + +- Requires Cloudflare API token in .env file +- Maximum 5 retry attempts to prevent infinite loops +- Waits 3 minutes for in-progress deployments +- Chrome MCP must be running for site testing +- Tests both deployment success AND actual functionality diff --git a/.opencode/skills/Utilities/CreateCLI/FrameworkComparison.md b/.opencode/skills/Utilities/CreateCLI/FrameworkComparison.md new file mode 100755 index 00000000..abaca87c --- /dev/null +++ b/.opencode/skills/Utilities/CreateCLI/FrameworkComparison.md @@ -0,0 +1,477 @@ +# CLI Framework Comparison + +**Comprehensive analysis of TypeScript CLI frameworks for informed tier selection** + +--- + +## 🎯 Quick Recommendation Matrix + +| Use Case | Framework | Why | +|----------|-----------|-----| +| **API Client** (2-10 commands) | Manual Parsing (Tier 1) | Zero deps, 300 lines, production-ready | +| **File Processor** (simple args) | Manual Parsing (Tier 1) | Fast development, type-safe, composable | +| **Multi-Tool** (10+ commands) | Commander.js (Tier 2) | Subcommands, auto-help, proven | +| **Plugin System** (extensible) | oclif (Tier 3) | Enterprise-grade, reference only | + +**Rule:** Default to Manual → escalate to Commander → reference oclif only + +--- + +## 📊 Framework Comparison Table + +| Framework | Stars | Bundle Size | TypeScript | Best For | Tier | +|-----------|-------|-------------|------------|----------|----------| +| **Manual Parsing** | N/A | 0 KB | Native | Simple CLIs (llcli) | Tier 1 ⭐ DEFAULT | +| **Commander.js** | 25K+ | ~100 KB | Built-in | General CLIs | Tier 2 | +| **oclif** | 12K+ | 22+ MB | First-class | Enterprise plugins | Tier 3 (ref only) | +| **cleye** | N/A | Small | Schema inference | Modern TS CLIs | Alternative | +| **citty** | N/A | Moderate | Discriminated unions | Complex type safety | Alternative | +| **Yargs** | 30K+ | Larger | @types | Config-heavy | Not recommended | + +--- + +## 1️⃣ TIER 1: Manual Parsing (llcli Pattern) + +### Pattern + +```typescript +#!/usr/bin/env bun + +async function main() { + const args = process.argv.slice(2); + + if (args.length === 0 || args[0] === '--help') { + showHelp(); + return; + } + + const command = args[0]; + + switch (command) { + case 'today': + await fetchToday(); + break; + case 'date': + if (!args[1]) { + console.error('Error: date requires YYYY-MM-DD argument'); + process.exit(1); + } + await fetchDate(args[1]); + break; + case 'search': + const keyword = args[1]; + const limitIdx = args.indexOf('--limit'); + const limit = limitIdx !== -1 ? parseInt(args[limitIdx + 1]) : 20; + await fetchSearch(keyword, limit); + break; + default: + console.error(`Unknown command: ${command}`); + process.exit(1); + } +} + +main().catch(error => { + console.error('Fatal:', error); + process.exit(1); +}); +``` + +### Pros +- ✅ Zero dependencies (no node_modules bloat) +- ✅ Complete control over parsing logic +- ✅ Type-safe with TypeScript interfaces +- ✅ 300-400 lines total (easy to understand) +- ✅ Fast development (no framework learning curve) +- ✅ Proven pattern (llcli is production-ready) +- ✅ Perfect for Bun runtime +- ✅ Deterministic behavior + +### Cons +- ❌ Manual help text (but this ensures quality) +- ❌ Manual argument parsing (but simple) +- ❌ No built-in subcommand routing (use Tier 2 if needed) +- ❌ Repetitive for 20+ commands (escalate at that point) + +### When to Use (DEFAULT) +- ✅ 2-10 commands +- ✅ API client wrappers +- ✅ Data transformers +- ✅ File processors +- ✅ Simple automation tools +- ✅ JSON output only +- ✅ Fast development priority + +### Reference Implementation +**Location:** `~/.opencode/Bin/llcli/llcli.ts` (327 lines) +**Commands:** today, date, search +**Pattern:** Exactly what this tier generates + +--- + +## 2️⃣ TIER 2: Commander.js + +### Pattern + +```typescript +#!/usr/bin/env bun + +import { Command } from 'commander'; + +const program = new Command(); + +program + .name('mycli') + .description('Production CLI tool') + .version('1.0.0'); + +program + .command('convert ') + .option('-o, --output ', 'output file') + .option('--verbose', 'verbose logging') + .action((format: string, input: string, options) => { + console.log(`Converting ${input} to ${format}`); + if (options.output) { + console.log(`Output: ${options.output}`); + } + }); + +program + .command('validate') + .argument('', 'file to validate') + .option('--strict', 'strict mode') + .action((file: string, options) => { + console.log(`Validating ${file}`); + }); + +program.parse(); +``` + +### Pros +- ✅ Auto-generated help (from command definitions) +- ✅ Subcommand routing built-in +- ✅ Fluent API (readable, chainable) +- ✅ TypeScript definitions included +- ✅ Large community (25K+ stars) +- ✅ Well-documented +- ✅ Option parsing automatic +- ✅ Lightweight (~100 KB, zero sub-dependencies) + +### Cons +- ❌ Framework dependency (not zero-dep like Tier 1) +- ❌ Learning curve (need to understand API) +- ❌ Opinionated structure +- ❌ Overkill for simple CLIs (use Tier 1 instead) +- ❌ Bun may prefer zero-dep approach + +### When to Use (ESCALATION) +- ❌ 10+ commands needing organization +- ❌ Subcommands (e.g., `cli convert json csv` vs `cli convert csv json`) +- ❌ Plugin architecture needed +- ❌ Complex option combinations +- ❌ Multiple output format engines +- ❌ Git-style command groups + +### Example Use Case +```bash +# Data transformation CLI with subcommands +data-cli convert json csv input.json --output data.csv +data-cli convert csv json input.csv +data-cli validate schema data.json --strict +data-cli analyze stats data.csv +data-cli analyze trends data.csv --window 7d +``` + +**Pattern:** Commands naturally group into categories (convert, validate, analyze) + +--- + +## 3️⃣ TIER 3: oclif (Reference Only) + +### Pattern + +```typescript +import { Command, Flags, Args } from '@oclif/core'; + +export default class Hello extends Command { + static description = 'Say hello'; + + static examples = [ + '<%= config.bin %> <%= command.id %> --name World', + ]; + + static flags = { + name: Flags.string({ + char: 'n', + description: 'name to greet', + required: true, + }), + verbose: Flags.boolean({ char: 'v' }), + }; + + static args = { + file: Args.string({ description: 'file to process' }), + }; + + async run() { + const { flags, args } = await this.parse(Hello); + this.log(`Hello ${flags.name}!`); + } +} +``` + +### Pros +- ✅ Enterprise-grade plugin system +- ✅ Code generation (`oclif generate command`) +- ✅ Topics for hierarchical commands +- ✅ Auto-updates mechanism +- ✅ Multi-command CLIs (Heroku, Salesforce scale) +- ✅ Class-based commands (OOP style) +- ✅ ES modules + CommonJS compatible + +### Cons +- ❌ Heavy bundle size (22+ MB) +- ❌ Steep learning curve +- ❌ Complex setup +- ❌ Overkill for 99% of CLIs +- ❌ Not aligned with PAI's minimal approach + +### When to Reference (RARE) +- Enterprise plugin systems (Heroku CLI scale) +- 50+ commands with complex organization +- Auto-update mechanisms critical +- Multi-tenant CLI platforms + +**Note:** This skill does NOT generate oclif CLIs. Documentation only for reference. + +--- + +## 🔬 RESEARCH FINDINGS: Type-Safe Frameworks + +### cleye (Schema-Driven Inference) + +**Pattern:** +```typescript +import { cli } from 'cleye'; + +const argv = cli({ + name: 'mycli', + flags: { + noCache: { + type: Boolean, + description: 'Disable cache', + }, + tsconfig: { + type: String, + description: 'Path to tsconfig', + }, + }, + parameters: [' + + diff --git a/.opencode/skills/Utilities/Parser/Web/index.html b/.opencode/skills/Utilities/Parser/Web/index.html new file mode 100755 index 00000000..7296b66d --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Web/index.html @@ -0,0 +1,66 @@ + + + + + + Content Parser + + + +
+ +
+

Content Parser

+

Extract structured data from URLs for newsletter database

+
+ + +
+

Parse Content

+
+ + +
+
+ + +
+
+ + + + + + +
+ + + + diff --git a/.opencode/skills/Utilities/Parser/Web/parser.js b/.opencode/skills/Utilities/Parser/Web/parser.js new file mode 100755 index 00000000..d0fa4751 --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Web/parser.js @@ -0,0 +1,526 @@ +// Content Parser Web App +class ParserApp { + constructor() { + this.parseBtn = document.getElementById('parse-btn'); + this.loadBtn = document.getElementById('load-btn'); + this.urlInput = document.getElementById('url-input'); + this.progressSection = document.getElementById('progress-section'); + this.progressContainer = document.getElementById('progress-container'); + this.resultsSection = document.getElementById('results-section'); + this.resultsContainer = document.getElementById('results-container'); + this.successCount = document.getElementById('success-count'); + this.failedCount = document.getElementById('failed-count'); + + this.results = []; + this.successfulCount = 0; + this.failedCount = 0; + + this.init(); + } + + init() { + this.parseBtn.addEventListener('click', () => this.handleParse()); + this.loadBtn.addEventListener('click', () => this.handleLoadExisting()); + + // Allow Ctrl+Enter to submit + this.urlInput.addEventListener('keydown', (e) => { + if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') { + this.handleParse(); + } + }); + } + + async handleLoadExisting() { + try { + // Reset state + this.results = []; + this.successfulCount = 0; + this.failedCount = 0; + this.progressContainer.innerHTML = ''; + this.resultsContainer.innerHTML = ''; + this.progressSection.style.display = 'none'; + + // Disable button + this.loadBtn.disabled = true; + this.loadBtn.innerHTML = '⏳ Loading...'; + + // Load index of available files + const indexResponse = await fetch('output/index.json'); + if (!indexResponse.ok) { + throw new Error('Failed to load file index'); + } + + const fileList = await indexResponse.json(); + + // Load each JSON file + for (const filename of fileList) { + try { + const response = await fetch(`output/${filename}`); + if (response.ok) { + const data = await response.json(); + this.results.push(data); + this.successfulCount++; + } else { + this.failedCount++; + } + } catch (error) { + console.error(`Failed to load ${filename}:`, error); + this.failedCount++; + } + } + + // Re-enable button + this.loadBtn.disabled = false; + this.loadBtn.innerHTML = '📂 Load Existing Results'; + + // Show results + this.showResults(); + } catch (error) { + this.loadBtn.disabled = false; + this.loadBtn.innerHTML = '📂 Load Existing Results'; + this.showError('Failed to load existing results: ' + error.message); + } + } + + async handleParse() { + const urls = this.getURLs(); + + if (urls.length === 0) { + this.showError('Please enter at least one URL'); + return; + } + + // Reset state + this.results = []; + this.successfulCount = 0; + this.failedCount = 0; + this.progressContainer.innerHTML = ''; + this.resultsContainer.innerHTML = ''; + + // Show progress section + this.progressSection.style.display = 'block'; + this.resultsSection.style.display = 'none'; + + // Disable button + this.parseBtn.disabled = true; + this.parseBtn.innerHTML = '⏳ Parsing...'; + + // Process each URL + for (let i = 0; i < urls.length; i++) { + await this.parseURL(urls[i], i + 1, urls.length); + } + + // Re-enable button + this.parseBtn.disabled = false; + this.parseBtn.innerHTML = '⚡ Parse URLs'; + + // Show results + this.showResults(); + } + + getURLs() { + const text = this.urlInput.value.trim(); + if (!text) return []; + + return text + .split('\n') + .map(url => url.trim()) + .filter(url => url && this.isValidURL(url)); + } + + isValidURL(string) { + try { + new URL(string); + return true; + } catch (_) { + return false; + } + } + + async parseURL(url, index, total) { + const progressItem = this.createProgressItem(url, index, total); + this.progressContainer.appendChild(progressItem); + + try { + // Detect content type + this.updateProgressStatus(progressItem, 'Detecting content type...', 'active'); + await this.delay(800); + + const contentType = this.detectContentType(url); + this.updateProgressStatus(progressItem, `Type: ${contentType}`, 'active'); + await this.delay(600); + + // Fetch content + this.updateProgressStatus(progressItem, 'Fetching content...', 'active'); + await this.delay(1200); + + // Extract entities + this.updateProgressStatus(progressItem, 'Extracting entities...', 'active'); + await this.delay(1500); + + // Analyze content + this.updateProgressStatus(progressItem, 'Analyzing content...', 'active'); + await this.delay(1000); + + // Generate JSON + this.updateProgressStatus(progressItem, 'Generating JSON...', 'active'); + await this.delay(800); + + // Create mock result + const result = this.createMockResult(url, contentType); + this.results.push(result); + this.successfulCount++; + + this.updateProgressStatus(progressItem, '✓ Completed successfully', 'success'); + progressItem.classList.remove('active'); + progressItem.classList.add('success'); + + } catch (error) { + this.failedCount++; + this.updateProgressStatus(progressItem, `✗ Error: ${error.message}`, 'error'); + progressItem.classList.remove('active'); + progressItem.classList.add('error'); + } + } + + createProgressItem(url, index, total) { + const item = document.createElement('div'); + item.className = 'progress-item'; + item.innerHTML = ` + +
+
${this.escapeHtml(url)}
+
Processing ${index} of ${total}...
+
+ `; + return item; + } + + updateProgressStatus(item, status, type) { + const statusEl = item.querySelector('.progress-status'); + const iconEl = item.querySelector('.progress-icon'); + + statusEl.textContent = status; + + if (type === 'success') { + iconEl.textContent = '✓'; + iconEl.classList.remove('spinner'); + } else if (type === 'error') { + iconEl.textContent = '✗'; + iconEl.classList.remove('spinner'); + } + } + + detectContentType(url) { + if (url.includes('youtube.com') || url.includes('youtu.be')) { + return 'video'; + } else if (url.includes('twitter.com') || url.includes('x.com')) { + return 'tweet_thread'; + } else if (url.endsWith('.pdf')) { + return 'pdf'; + } else if (url.includes('substack.com') || url.includes('beehiiv.com')) { + return 'newsletter'; + } else { + return 'article'; + } + } + + createMockResult(url, contentType) { + const timestamp = new Date().toISOString(); + const uuid = this.generateUUID(); + const domain = new URL(url).hostname; + + return { + content: { + id: uuid, + type: contentType, + title: this.generateMockTitle(url), + summary: { + short: "This is a mock short summary of the content (1-2 sentences).", + medium: "This is a mock medium summary that provides more detail about the content. It includes the main points and key takeaways from the parsed content.", + long: "This is a comprehensive mock summary with multiple paragraphs. It covers all the major themes, arguments, and conclusions from the source material. This would typically be much longer and more detailed in a real parsing scenario." + }, + content: { + full_text: "Mock full text content extracted from the source...", + transcript: contentType === 'video' ? "Mock transcript..." : null, + excerpts: [ + "Notable quote or excerpt from the content", + "Another important point or statistic" + ] + }, + metadata: { + source_url: url, + canonical_url: url, + published_date: new Date(Date.now() - Math.random() * 30 * 24 * 60 * 60 * 1000).toISOString(), + accessed_date: timestamp, + language: "en", + word_count: Math.floor(Math.random() * 3000) + 500, + read_time_minutes: Math.floor(Math.random() * 15) + 3, + author_platform: this.getAuthorPlatform(contentType) + } + }, + people: this.generateMockPeople(), + companies: this.generateMockCompanies(), + topics: { + primary_category: "AI", + secondary_categories: ["technology", "business"], + tags: ["artificial-intelligence", "machine-learning", "innovation"], + keywords: ["AI", "technology", "innovation", "future"], + themes: ["AI advancement", "technological change"], + newsletter_sections: ["Headlines", "Analysis"] + }, + links: [ + { + url: url, + domain: domain, + title: "Source article", + description: "Original source material", + link_type: "source", + context: "Main content source", + position: "beginning" + } + ], + sources: [], + newsletter_metadata: { + issue_number: null, + section: null, + position_in_section: null, + editorial_note: null, + include_in_newsletter: false, + scheduled_date: null + }, + analysis: { + sentiment: "neutral", + importance_score: Math.floor(Math.random() * 5) + 5, + novelty_score: Math.floor(Math.random() * 5) + 5, + controversy_score: Math.floor(Math.random() * 5) + 3, + relevance_to_audience: ["technologists", "ai_researchers"], + key_insights: [ + "Key insight 1 from the analysis", + "Key insight 2 from the analysis", + "Key insight 3 from the analysis" + ], + related_content_ids: [], + trending_potential: ["low", "medium", "high"][Math.floor(Math.random() * 3)] + }, + extraction_metadata: { + processed_date: timestamp, + processing_method: "hybrid", + confidence_score: 0.7 + Math.random() * 0.25, + warnings: [], + version: "1.0.0" + } + }; + } + + generateMockTitle(url) { + const domain = new URL(url).hostname; + const titles = [ + "Breaking: Major AI Advancement Announced", + "Understanding Modern Machine Learning Systems", + "The Future of Artificial Intelligence", + "New Research Breakthrough in Neural Networks", + "Industry Analysis: AI Market Trends" + ]; + return titles[Math.floor(Math.random() * titles.length)] + ` (from ${domain})`; + } + + generateMockPeople() { + return [ + { + name: "John Doe", + role: "author", + title: "Senior Researcher", + company: "Tech Corp", + social: { + twitter: "@johndoe", + linkedin: "linkedin.com/in/johndoe", + email: null, + website: "johndoe.com" + }, + context: "Article author, expert in AI", + importance: "primary" + } + ]; + } + + generateMockCompanies() { + return [ + { + name: "OpenAI", + domain: "openai.com", + industry: "AI", + context: "Main subject of article", + mentioned_as: "subject", + sentiment: "positive" + } + ]; + } + + getAuthorPlatform(contentType) { + const platforms = { + video: 'youtube', + tweet_thread: 'twitter', + pdf: 'arxiv', + newsletter: 'substack', + article: 'blog' + }; + return platforms[contentType] || 'other'; + } + + generateUUID() { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + const r = Math.random() * 16 | 0; + const v = c == 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); + } + + showResults() { + console.log('showResults called, results count:', this.results.length); + this.resultsSection.style.display = 'block'; + this.successCount.textContent = `✓ ${this.successfulCount} successful`; + this.failedCount.textContent = `✗ ${this.failedCount} failed`; + + console.log('Results container:', this.resultsContainer); + this.results.forEach((result, index) => { + try { + console.log('Creating result item', index, result.content.title); + const resultItem = this.createResultItem(result, index); + console.log('Result item created:', resultItem); + this.resultsContainer.appendChild(resultItem); + console.log('Result item appended'); + } catch (error) { + console.error('Error creating result item:', error); + console.error('Result data:', result); + } + }); + console.log('Results rendered, container children:', this.resultsContainer.children.length); + } + + createResultItem(result, index) { + const item = document.createElement('div'); + item.className = 'result-item'; + + const filename = this.generateFilename(result.content.title); + + item.innerHTML = ` +
+
+

${this.escapeHtml(result.content.title)}

+

${filename}

+
+
+ + +
+ ▼ +
+
+ 
${this.syntaxHighlight(JSON.stringify(result, null, 2))}
+
+ `; + + // Add event listeners + const header = item.querySelector('.result-header'); + header.addEventListener('click', (e) => { + if (!e.target.closest('button')) { + this.toggleJSON(index); + } + }); + + const downloadBtn = item.querySelector('.download-btn'); + downloadBtn.addEventListener('click', (e) => { + e.stopPropagation(); + this.downloadJSON(result, filename); + }); + + const copyBtn = item.querySelector('.copy-btn'); + copyBtn.addEventListener('click', (e) => { + e.stopPropagation(); + this.copyJSON(result); + }); + + return item; + } + + generateFilename(title) { + const timestamp = new Date().toISOString().replace(/:/g, '-').split('.')[0]; + const sanitized = title + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .substring(0, 50); + return `${timestamp}_${sanitized}.json`; + } + + toggleJSON(index) { + const viewer = document.querySelector(`.json-viewer[data-index="${index}"]`); + const icon = document.querySelector(`.result-header[data-index="${index}"] .expand-icon`); + + viewer.classList.toggle('expanded'); + icon.classList.toggle('expanded'); + } + + downloadJSON(data, filename) { + const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } + + async copyJSON(data) { + try { + await navigator.clipboard.writeText(JSON.stringify(data, null, 2)); + // Could add a toast notification here + } catch (err) { + console.error('Failed to copy:', err); + } + } + + syntaxHighlight(json) { + return json.replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?)/g, (match) => { + let cls = 'json-number'; + if (/^"/.test(match)) { + if (/:$/.test(match)) { + cls = 'json-key'; + } else { + cls = 'json-string'; + } + } else if (/true|false/.test(match)) { + cls = 'json-boolean'; + } else if (/null/.test(match)) { + cls = 'json-null'; + } + return `${match}`; + }); + } + + showError(message) { + alert(message); // Could be replaced with a nicer toast notification + } + + escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; + } + + delay(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); + } +} + +// Initialize app when DOM is ready +document.addEventListener('DOMContentLoaded', () => { + new ParserApp(); +}); diff --git a/.opencode/skills/Utilities/Parser/Web/simple-test.html b/.opencode/skills/Utilities/Parser/Web/simple-test.html new file mode 100755 index 00000000..7564749f --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Web/simple-test.html @@ -0,0 +1,62 @@ + + + + + + Simple Test + + + +

Simple Load Test

+ +
+ + + + diff --git a/.opencode/skills/Utilities/Parser/Web/styles.css b/.opencode/skills/Utilities/Parser/Web/styles.css new file mode 100755 index 00000000..2eeee82f --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Web/styles.css @@ -0,0 +1,498 @@ +/* Tokyo Night Storm Theme Variables */ +:root { + /* Colors */ + --primary: #9d7cd8; + --primary-hover: #b39bdf; + --background: #24283b; + --foreground: #c0caf5; + --card-bg: #1f2335; + --border: #414868; + --success: #9ece6a; + --warning: #e0af68; + --error: #f7768e; + --info: #7aa2f7; + + /* Spacing (8px grid) */ + --space-1: 4px; + --space-2: 8px; + --space-3: 12px; + --space-4: 16px; + --space-6: 24px; + --space-8: 32px; + --space-12: 48px; + --space-16: 64px; + + /* Typography */ + --font-sans: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', sans-serif; + --font-mono: 'SF Mono', 'Monaco', 'Inconsolata', 'Fira Code', monospace; + + /* Transitions */ + --transition: 200ms cubic-bezier(0.4, 0, 0.2, 1); +} + +/* Reset & Base */ +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: var(--font-sans); + background: var(--background); + color: var(--foreground); + line-height: 1.5; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + min-height: 100vh; + padding: var(--space-6); +} + +/* Container */ +.container { + max-width: 1280px; + margin: 0 auto; +} + +/* Header */ +.header { + text-align: center; + margin-bottom: var(--space-12); + padding: var(--space-8) 0; +} + +.header h1 { + font-size: clamp(32px, 5vw, 48px); + font-weight: 700; + color: var(--primary); + margin-bottom: var(--space-2); + line-height: 1.2; +} + +.subtitle { + font-size: 16px; + color: var(--foreground); + opacity: 0.8; +} + +/* Card Component */ +.card { + background: var(--card-bg); + border-radius: 12px; + border: 1px solid var(--border); + padding: var(--space-6); + margin-bottom: var(--space-6); + transition: border-color var(--transition); +} + +.card:hover { + border-color: var(--primary); +} + +.card h2 { + font-size: 24px; + font-weight: 600; + margin-bottom: var(--space-4); + color: var(--foreground); +} + +/* Form Elements */ +.form-group { + margin-bottom: var(--space-4); +} + +.form-group label { + display: block; + font-size: 14px; + font-weight: 500; + margin-bottom: var(--space-2); + color: var(--foreground); +} + +.url-input { + width: 100%; + background: var(--background); + border: 1px solid var(--border); + border-radius: 6px; + padding: var(--space-3) var(--space-4); + font-size: 16px; + font-family: var(--font-mono); + color: var(--foreground); + resize: vertical; + transition: border-color var(--transition); +} + +.url-input:focus { + outline: none; + border-color: var(--primary); + box-shadow: 0 0 0 3px rgba(157, 124, 216, 0.1); +} + +.url-input::placeholder { + color: var(--foreground); + opacity: 0.4; +} + +/* Button Group */ +.button-group { + display: flex; + gap: var(--space-3); + flex-wrap: wrap; +} + +/* Buttons */ +.btn-primary, +.btn-secondary { + border: none; + border-radius: 8px; + padding: var(--space-3) var(--space-6); + font-size: 16px; + font-weight: 500; + cursor: pointer; + transition: all var(--transition); + display: inline-flex; + align-items: center; + gap: var(--space-2); + min-height: 44px; +} + +.btn-primary { + background: var(--primary); + color: #ffffff; +} + +.btn-primary:hover { + background: var(--primary-hover); + transform: translateY(-1px); + box-shadow: 0 4px 12px rgba(157, 124, 216, 0.3); +} + +.btn-secondary { + background: transparent; + border: 2px solid var(--primary); + color: var(--primary); +} + +.btn-secondary:hover { + background: rgba(157, 124, 216, 0.1); + transform: translateY(-1px); +} + +.btn-primary:active, +.btn-secondary:active { + transform: translateY(0); +} + +.btn-primary:disabled, +.btn-secondary:disabled { + opacity: 0.5; + cursor: not-allowed; + transform: none; +} + +.btn-icon { + font-size: 18px; +} + +/* Progress Section */ +.progress-container { + display: flex; + flex-direction: column; + gap: var(--space-3); +} + +.progress-item { + background: var(--background); + border: 1px solid var(--border); + border-radius: 8px; + padding: var(--space-4); + display: flex; + align-items: center; + gap: var(--space-3); + transition: all var(--transition); +} + +.progress-item.active { + border-color: var(--info); + box-shadow: 0 0 0 2px rgba(122, 162, 247, 0.1); +} + +.progress-item.success { + border-color: var(--success); +} + +.progress-item.error { + border-color: var(--error); +} + +.progress-icon { + font-size: 20px; + flex-shrink: 0; +} + +.progress-icon.spinner { + animation: spin 1s linear infinite; +} + +@keyframes spin { + from { transform: rotate(0deg); } + to { transform: rotate(360deg); } +} + +.progress-content { + flex: 1; + min-width: 0; +} + +.progress-url { + font-size: 14px; + font-family: var(--font-mono); + color: var(--foreground); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + margin-bottom: var(--space-1); +} + +.progress-status { + font-size: 12px; + color: var(--foreground); + opacity: 0.7; +} + +/* Results Section */ +.results-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: var(--space-4); + flex-wrap: wrap; + gap: var(--space-3); +} + +.results-stats { + display: flex; + gap: var(--space-4); +} + +.stat { + font-size: 14px; + font-weight: 500; + padding: var(--space-1) var(--space-3); + border-radius: 6px; +} + +.stat.success { + color: var(--success); + background: rgba(158, 206, 106, 0.1); +} + +.stat.failed { + color: var(--error); + background: rgba(247, 118, 142, 0.1); +} + +/* Result Item */ +.result-item { + background: var(--background); + border: 1px solid var(--border); + border-radius: 8px; + margin-bottom: var(--space-3); + overflow: hidden; +} + +.result-header { + padding: var(--space-4); + display: flex; + justify-content: space-between; + align-items: center; + gap: var(--space-3); + cursor: pointer; + transition: background var(--transition); +} + +.result-header:hover { + background: rgba(157, 124, 216, 0.05); +} + +.result-title { + flex: 1; + min-width: 0; +} + +.result-title h3 { + font-size: 16px; + font-weight: 500; + color: var(--foreground); + margin-bottom: var(--space-1); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.result-title p { + font-size: 12px; + font-family: var(--font-mono); + color: var(--foreground); + opacity: 0.6; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.result-actions { + display: flex; + gap: var(--space-2); +} + +.btn-small { + padding: var(--space-1) var(--space-3); + font-size: 12px; + background: var(--primary); + color: #ffffff; + border: none; + border-radius: 4px; + cursor: pointer; + transition: all var(--transition); + min-height: 32px; +} + +.btn-small:hover { + background: var(--primary-hover); +} + +.expand-icon { + font-size: 18px; + color: var(--foreground); + opacity: 0.5; + transition: transform var(--transition); +} + +.expand-icon.expanded { + transform: rotate(180deg); +} + +/* JSON Viewer */ +.json-viewer { + padding: var(--space-4); + background: var(--background); + border-top: 1px solid var(--border); + max-height: 0; + overflow: hidden; + transition: max-height var(--transition); +} + +.json-viewer.expanded { + max-height: 600px; + overflow-y: auto; +} + +.json-viewer pre { + font-family: var(--font-mono); + font-size: 13px; + line-height: 1.6; + color: var(--foreground); + white-space: pre-wrap; + word-break: break-word; +} + +/* Syntax Highlighting */ +.json-key { + color: var(--info); +} + +.json-string { + color: var(--success); +} + +.json-number { + color: var(--warning); +} + +.json-boolean { + color: var(--primary); +} + +.json-null { + color: var(--error); +} + +/* Scrollbar Styling */ +::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +::-webkit-scrollbar-track { + background: var(--background); +} + +::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 4px; +} + +::-webkit-scrollbar-thumb:hover { + background: var(--primary); +} + +/* Responsive Design */ +@media (max-width: 640px) { + body { + padding: var(--space-4); + } + + .header h1 { + font-size: 28px; + } + + .card { + padding: var(--space-4); + } + + .results-header { + flex-direction: column; + align-items: flex-start; + } + + .result-header { + flex-direction: column; + align-items: flex-start; + } + + .result-actions { + width: 100%; + } + + .btn-small { + flex: 1; + } +} + +/* Accessibility - Focus Indicators */ +button:focus-visible, +textarea:focus-visible { + outline: 2px solid var(--primary); + outline-offset: 2px; +} + +/* Loading State */ +.loading { + position: relative; + pointer-events: none; + opacity: 0.6; +} + +.loading::after { + content: ''; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + width: 24px; + height: 24px; + border: 3px solid var(--border); + border-top-color: var(--primary); + border-radius: 50%; + animation: spin 1s linear infinite; +} diff --git a/.opencode/skills/Utilities/Parser/Workflows/BatchEntityExtractionGemini3.md b/.opencode/skills/Utilities/Parser/Workflows/BatchEntityExtractionGemini3.md new file mode 100755 index 00000000..e615f765 --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Workflows/BatchEntityExtractionGemini3.md @@ -0,0 +1,757 @@ +# Batch Entity Extraction with Gemini 3 Pro + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the BatchEntityExtractionGemini3 workflow in the Parser skill to extract entities"}' \ + > /dev/null 2>&1 & +``` + +Running the **BatchEntityExtractionGemini3** workflow in the **Parser** skill to extract entities... + +## Purpose + +Extract structured entities from multiple URLs in a single batch request using Gemini 3 Pro's 1M context window for better entity recognition, relationship extraction, and cross-article entity resolution. + +## Gemini 3 Pro Advantages + +**Superior Entity Recognition:** +- Better identification of people, companies, and concepts +- Deep reasoning for relationship extraction +- Multimodal capability (extract from PDFs, images in articles) + +**Batch Processing:** +- 1M context window supports 10-50 URLs per request +- Faster than individual parsing +- Better cross-article entity resolution +- Consistent entity recognition across content + +**Enhanced Accuracy:** +- More accurate role/affiliation detection +- Better sentiment analysis +- Improved topic classification +- Superior relationship mapping + +## Integration with parser + +This workflow ENHANCES the existing parser: +- **Current**: Uses GeminiResearcher for individual extraction +- **New**: Uses Gemini 3 Pro directly for batch entity accuracy +- **Schema**: Compatible with `schema/content-schema.json` +- **Collision Detection**: Integrates with `entity-index.json` +- **Output**: Deterministic JSON with GUIDs + +## Workflow Steps + +### 1. URL Collection + +**Input**: List of URLs to parse + +```bash +# Interactive collection +urls=( + "https://example.com/article-1" + "https://example.com/article-2" + "https://example.com/article-3" +) + +# Or from file +urls=$(cat urls.txt) + +# Or from clipboard +urls=$(pbpaste) +``` + +**Pre-flight Check**: Check for already-parsed URLs + +```typescript +import { isContentAlreadyParsed } from '../utils/collision-detection'; + +const entityIndex = JSON.parse( + fs.readFileSync('~/.opencode/skills/parser/entity-index.json', 'utf-8') +); + +const newUrls = urls.filter(url => !isContentAlreadyParsed(url, entityIndex)); +console.log(`Skipping ${urls.length - newUrls.length} already-parsed URLs`); +console.log(`Processing ${newUrls.length} new URLs`); +``` + +### 2. Content Fetching + +**Option A: WebFetch (Simple)** +```bash +for url in "${urls[@]}"; do + echo "=== URL: $url ===" + curl -s "$url" | html2text + echo "---" +done > batch-content.txt +``` + +**Option B: Firecrawl (Better Quality)** +```typescript +import { WebFetch } from '@tool/webfetch'; + +const contents = await Promise.all( + urls.map(async (url) => { + const content = await WebFetch(url, "Extract all text content"); + return { + url, + content + }; + }) +); +``` + +**Option C: Bright Data (Production)** +```typescript +import { scrapeAsMarkdown } from '~/.opencode/skills/mcp/Providers/brightdata/actors'; + +const contents = await Promise.all( + urls.map(async (url) => { + const markdown = await scrapeAsMarkdown(url); + return { + url, + markdown + }; + }) +); +``` + +### 3. Batch Context Assembly + +Concatenate all content with clear separators for Gemini 3 Pro: + +```typescript +function assembleBatchContext(contents: Array<{url: string, content: string}>): string { + return contents.map((item, index) => ` +=========================================== +ARTICLE ${index + 1} of ${contents.length} +URL: ${item.url} +=========================================== + +${item.content} + +`).join('\n\n'); +} + +const batchContext = assembleBatchContext(contents); +``` + +**Context Size Management:** +- Target: 500K-800K tokens per batch +- Limit: 10-50 URLs depending on article length +- If exceeds: Split into multiple batches + +### 4. Gemini 3 Entity Extraction + +**Prompt Template:** + +```typescript +const extractionPrompt = `You are a precise entity extraction system. Extract structured entities from these ${contents.length} articles following the EXACT schema provided. + +CRITICAL REQUIREMENTS: +1. Extract ALL mentioned people, companies, links, and sources +2. Use EXACT field names from schema +3. Maintain consistency across articles (same entity = same name) +4. Provide rich context and relationships +5. Return VALID JSON only (no markdown, no explanations) + +SCHEMA (REQUIRED FIELDS): +{ + "articles": [ + { + "url": "original URL", + "content_id": "leave empty - will be assigned", + "people": [ + { + "id": "leave empty - will be assigned", + "name": "Full Name", + "role": "author|subject|mentioned|quoted|expert|interviewer|interviewee", + "title": "Job title or null", + "company": "Current company or null", + "social": { + "twitter": "@handle or null", + "linkedin": "URL or null", + "email": "email or null", + "website": "URL or null" + }, + "context": "Why mentioned in content", + "importance": "primary|secondary|minor" + } + ], + "companies": [ + { + "id": "leave empty - will be assigned", + "name": "Company Name", + "domain": "domain.com or null", + "industry": "AI|security|fintech|etc or null", + "context": "Relevance in content", + "mentioned_as": "subject|source|example|competitor|partner|acquisition|product|other", + "sentiment": "positive|neutral|negative|mixed" + } + ], + "links": [ + { + "id": "leave empty - will be assigned", + "url": "Full URL", + "domain": "domain.com", + "title": "Link text or null", + "description": "What link points to or null", + "link_type": "reference|source|related|tool|research|product|social|other", + "context": "Why included", + "position": "beginning|middle|end|sidebar|footer" + } + ], + "sources": [ + { + "id": "leave empty - will be assigned", + "publication": "Publication name or null", + "author": "Original author or null", + "url": "Source URL or null", + "published_date": "ISO 8601 date or null", + "source_type": "research_paper|news_article|blog_post|twitter_thread|podcast|video|book|other" + } + ], + "topics": { + "primary_category": "AI|security|tech|business|science", + "secondary_categories": ["related", "categories"], + "tags": ["specific", "tags"], + "keywords": ["important", "terms"], + "themes": ["broader", "concepts"], + "newsletter_sections": ["Headlines", "Analysis", "Tools", "AI", "Security"] + } + } + ] +} + +CROSS-ARTICLE ENTITY RESOLUTION: +- If same person appears in multiple articles, use IDENTICAL name +- If same company appears in multiple articles, use IDENTICAL name and domain +- Maintain entity consistency for better deduplication + +ARTICLES TO PROCESS: +${batchContext} + +Return ONLY the JSON object above with all entities extracted. No markdown, no explanations.`; +``` + +**Execute with Gemini 3 Pro:** + +```bash +# Using llm CLI with Gemini 3 Pro +llm -m gemini-3-pro-preview "$(cat extraction-prompt.txt)" > raw-entities.json + +# Or via API +curl https://generativelanguage.googleapis.com/v1/models/gemini-3-pro:generateContent \ + -H "Content-Type: application/json" \ + -H "x-goog-api-key: $GEMINI_API_KEY" \ + -d "{ + \"contents\": [{ + \"parts\": [{ + \"text\": \"$(cat extraction-prompt.txt)\" + }] + }], + \"generationConfig\": { + \"temperature\": 0.1, + \"maxOutputTokens\": 8192 + } + }" > raw-entities.json +``` + +**Temperature Settings:** +- `0.1` for deterministic entity extraction +- Low temperature = more consistent entity naming +- Better for deduplication + +### 5. Collision Detection & GUID Assignment + +Process the extracted entities and assign GUIDs: + +```typescript +import { v4 as uuidv4 } from 'uuid'; +import fs from 'fs'; + +interface EntityIndex { + version: string; + last_updated: string; + people: Record; + companies: Record; + links: Record; + sources: Record; +} + +function normalizeName(name: string): string { + return name.toLowerCase().trim(); +} + +function normalizeUrl(url: string): string { + return url.toLowerCase().trim().replace(/\/$/, ''); +} + +function normalizeDomain(domain: string | null): string | null { + return domain ? domain.toLowerCase().trim() : null; +} + +function getOrCreatePerson( + personData: any, + entityIndex: EntityIndex, + contentId: string +): string { + const canonicalId = normalizeName(personData.name); + + if (entityIndex.people[canonicalId]) { + // Reuse existing GUID + const existing = entityIndex.people[canonicalId]; + existing.occurrences++; + if (!existing.content_ids.includes(contentId)) { + existing.content_ids.push(contentId); + } + return existing.id; + } else { + // Create new GUID + const personId = uuidv4(); + entityIndex.people[canonicalId] = { + id: personId, + name: personData.name, + first_seen: new Date().toISOString(), + occurrences: 1, + content_ids: [contentId] + }; + return personId; + } +} + +function getOrCreateCompany( + companyData: any, + entityIndex: EntityIndex, + contentId: string +): string { + const canonicalId = companyData.domain + ? normalizeDomain(companyData.domain)! + : normalizeName(companyData.name); + + if (entityIndex.companies[canonicalId]) { + const existing = entityIndex.companies[canonicalId]; + existing.occurrences++; + if (!existing.content_ids.includes(contentId)) { + existing.content_ids.push(contentId); + } + return existing.id; + } else { + const companyId = uuidv4(); + entityIndex.companies[canonicalId] = { + id: companyId, + name: companyData.name, + domain: companyData.domain, + first_seen: new Date().toISOString(), + occurrences: 1, + content_ids: [contentId] + }; + return companyId; + } +} + +function getOrCreateLink( + linkData: any, + entityIndex: EntityIndex, + contentId: string +): string { + const canonicalId = normalizeUrl(linkData.url); + + if (entityIndex.links[canonicalId]) { + const existing = entityIndex.links[canonicalId]; + existing.occurrences++; + if (!existing.content_ids.includes(contentId)) { + existing.content_ids.push(contentId); + } + return existing.id; + } else { + const linkId = uuidv4(); + entityIndex.links[canonicalId] = { + id: linkId, + url: linkData.url, + first_seen: new Date().toISOString(), + occurrences: 1, + content_ids: [contentId] + }; + return linkId; + } +} + +function getOrCreateSource( + sourceData: any, + entityIndex: EntityIndex, + contentId: string +): string { + const canonicalId = sourceData.url + ? normalizeUrl(sourceData.url) + : `${normalizeName(sourceData.author || '')}|${normalizeName(sourceData.publication || '')}`; + + if (entityIndex.sources[canonicalId]) { + const existing = entityIndex.sources[canonicalId]; + existing.occurrences++; + if (!existing.content_ids.includes(contentId)) { + existing.content_ids.push(contentId); + } + return existing.id; + } else { + const sourceId = uuidv4(); + entityIndex.sources[canonicalId] = { + id: sourceId, + url: sourceData.url, + author: sourceData.author, + publication: sourceData.publication, + first_seen: new Date().toISOString(), + occurrences: 1, + content_ids: [contentId] + }; + return sourceId; + } +} + +function processArticleEntities( + articleData: any, + entityIndex: EntityIndex +): any { + // Generate content ID + const contentId = uuidv4(); + + // Process people + const people = articleData.people.map((person: any) => ({ + ...person, + id: getOrCreatePerson(person, entityIndex, contentId) + })); + + // Process companies + const companies = articleData.companies.map((company: any) => ({ + ...company, + id: getOrCreateCompany(company, entityIndex, contentId) + })); + + // Process links + const links = articleData.links.map((link: any) => ({ + ...link, + id: getOrCreateLink(link, entityIndex, contentId) + })); + + // Process sources + const sources = articleData.sources.map((source: any) => ({ + ...source, + id: getOrCreateSource(source, entityIndex, contentId) + })); + + return { + ...articleData, + content_id: contentId, + people, + companies, + links, + sources + }; +} + +// Main processing +const entityIndexPath = '~/.opencode/skills/parser/entity-index.json'; +const entityIndex: EntityIndex = JSON.parse(fs.readFileSync(entityIndexPath, 'utf-8')); + +const rawEntities = JSON.parse(fs.readFileSync('raw-entities.json', 'utf-8')); + +const processedArticles = rawEntities.articles.map((article: any) => + processArticleEntities(article, entityIndex) +); + +// Update entity index last_updated timestamp +entityIndex.last_updated = new Date().toISOString(); + +// Save updated entity index +fs.writeFileSync(entityIndexPath, JSON.stringify(entityIndex, null, 2)); + +console.log(`Processed ${processedArticles.length} articles`); +console.log(`Entity index updated with new entries`); +``` + +### 6. Schema Validation + +Validate each processed article against the schema: + +```typescript +import Ajv from 'ajv'; +import addFormats from 'ajv-formats'; + +const ajv = new Ajv({ strict: false }); +addFormats(ajv); + +const schema = JSON.parse( + fs.readFileSync('~/.opencode/skills/parser/schema/content-schema.json', 'utf-8') +); + +const validate = ajv.compile(schema); + +processedArticles.forEach((article: any, index: number) => { + // Transform to full schema format + const fullContent = { + content: { + id: article.content_id, + type: "article", // Or detect from URL + title: article.title || "Untitled", + summary: { + short: article.summary?.short || "", + medium: article.summary?.medium || "", + long: article.summary?.long || "" + }, + content: { + full_text: article.full_text || null, + transcript: null, + excerpts: article.excerpts || [] + }, + metadata: { + source_url: article.url, + canonical_url: article.url, + published_date: article.published_date || null, + accessed_date: new Date().toISOString(), + language: "en", + word_count: article.word_count || null, + read_time_minutes: article.read_time_minutes || null, + author_platform: "blog" // Or detect + } + }, + people: article.people, + companies: article.companies, + topics: article.topics, + links: article.links, + sources: article.sources, + newsletter_metadata: { + issue_number: null, + section: null, + position_in_section: null, + editorial_note: null, + include_in_newsletter: false, + scheduled_date: null + }, + analysis: { + sentiment: "neutral", + importance_score: 5, + novelty_score: 5, + controversy_score: 1, + relevance_to_audience: ["general_tech"], + key_insights: article.key_insights || [], + related_content_ids: [], + trending_potential: "medium" + }, + extraction_metadata: { + processed_date: new Date().toISOString(), + processing_method: "gemini", + confidence_score: 0.9, + warnings: [], + version: "1.0.0" + } + }; + + const valid = validate(fullContent); + + if (!valid) { + console.error(`Article ${index + 1} validation failed:`, validate.errors); + } else { + // Save validated content + const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, -5); + const filename = `~/.opencode/skills/parser/output/${timestamp}_batch-${index + 1}.json`; + fs.writeFileSync(filename, JSON.stringify(fullContent, null, 2)); + console.log(`Saved: ${filename}`); + } +}); +``` + +### 7. Entity Index Update + +The entity index is automatically updated during step 5. Verify the update: + +```bash +# Check entity counts +cat ~/.opencode/skills/parser/entity-index.json | jq '{ + people_count: (.people | length), + companies_count: (.companies | length), + links_count: (.links | length), + sources_count: (.sources | length), + last_updated: .last_updated +}' +``` + +## Command-Line Integration + +### Complete Batch Processing Script + +```bash +#!/usr/bin/env bash +# batch-extract.sh - Batch entity extraction with Gemini 3 Pro + +set -euo pipefail + +# Check for URL input +if [ $# -eq 0 ]; then + echo "Usage: $0 ... " + echo " or: cat urls.txt | xargs $0" + exit 1 +fi + +URLS=("$@") +SKILL_DIR="$HOME/.claude/skills/parser" +OUTPUT_DIR="$SKILL_DIR/output" +TIMESTAMP=$(date +%Y%m%d-%H%M%S) + +echo "Processing ${#URLS[@]} URLs..." + +# Fetch content +echo "Fetching content..." +BATCH_CONTENT="" +for i in "${!URLS[@]}"; do + url="${URLS[$i]}" + echo "Fetching: $url" + content=$(curl -s "$url" | html2text) + BATCH_CONTENT+=" +=========================================== +ARTICLE $((i+1)) of ${#URLS[@]} +URL: $url +=========================================== + +$content + +" +done + +# Create extraction prompt +cat > /tmp/extraction-prompt.txt < /tmp/raw-entities.json + +# Process entities and assign GUIDs +echo "Processing entities and assigning GUIDs..." +bun run "$SKILL_DIR/utils/process-batch.ts" /tmp/raw-entities.json + +echo "Batch processing complete!" +echo "Results saved to: $OUTPUT_DIR" +``` + +### TypeScript CLI Tool + +```typescript +#!/usr/bin/env bun +// utils/process-batch.ts + +import { processArticleEntities } from './collision-detection'; +import fs from 'fs'; + +const rawEntitiesPath = process.argv[2]; +if (!rawEntitiesPath) { + console.error('Usage: process-batch.ts '); + process.exit(1); +} + +const entityIndexPath = `${process.env.HOME}/.claude/skills/parser/entity-index.json`; +const entityIndex = JSON.parse(fs.readFileSync(entityIndexPath, 'utf-8')); + +const rawEntities = JSON.parse(fs.readFileSync(rawEntitiesPath, 'utf-8')); + +const processedArticles = rawEntities.articles.map((article: any) => + processArticleEntities(article, entityIndex) +); + +// Update entity index +entityIndex.last_updated = new Date().toISOString(); +fs.writeFileSync(entityIndexPath, JSON.stringify(entityIndex, null, 2)); + +// Save processed articles +processedArticles.forEach((article: any, index: number) => { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, -5); + const filename = `${process.env.HOME}/.claude/skills/parser/output/${timestamp}_batch-${index + 1}.json`; + fs.writeFileSync(filename, JSON.stringify(article, null, 2)); + console.log(`Saved: ${filename}`); +}); + +console.log(`Processed ${processedArticles.length} articles`); +``` + +## Performance Characteristics + +**Single Article (Current):** +- Time: ~30-60 seconds per URL +- Cost: 1 API call per URL +- Quality: Good entity recognition + +**Batch Processing (Gemini 3 Pro):** +- Time: ~60-120 seconds for 10-50 URLs +- Cost: 1 API call for entire batch +- Quality: Better cross-article entity resolution +- Speedup: 10-50x faster +- Cost Savings: 90-98% reduction + +**Example: 20 URLs** +- Old: 20 API calls × 45s = 15 minutes +- New: 1 API call × 90s = 90 seconds +- Speedup: 10x faster +- Cost: 95% cheaper + +## Best Practices + +**Batch Size:** +- Start small: 5-10 URLs for testing +- Optimal: 10-20 URLs per batch +- Maximum: 50 URLs (stay under 1M context limit) + +**Entity Consistency:** +- Use low temperature (0.1) for deterministic output +- Include cross-article resolution instructions in prompt +- Review first batch output for consistency + +**Error Handling:** +- Validate schema compliance before saving +- Log validation errors for manual review +- Implement retry logic for API failures + +**Incremental Processing:** +- Check entity index before fetching content +- Skip already-parsed URLs +- Process new URLs only + +**Quality Control:** +- Spot-check first batch manually +- Verify GUID assignments are correct +- Ensure entity deduplication is working + +## Troubleshooting + +**Issue: Context too large** +- Solution: Reduce batch size or use excerpts instead of full content + +**Issue: Inconsistent entity naming** +- Solution: Lower temperature, add more examples to prompt + +**Issue: Missing entities** +- Solution: Adjust extraction prompt to be more explicit about requirements + +**Issue: GUID collisions** +- Solution: Check canonical ID normalization logic + +**Issue: Schema validation fails** +- Solution: Review required fields, ensure all are present + +## Future Enhancements + +1. **Automatic Batch Sizing**: Dynamically determine optimal batch size based on content length +2. **Parallel Batch Processing**: Process multiple batches in parallel +3. **Entity Enrichment**: Fetch additional data (LinkedIn profiles, company info) +4. **Fuzzy Deduplication**: Detect similar but not identical entities +5. **Knowledge Graph Visualization**: Graph relationships between entities +6. **Newsletter Integration**: Auto-suggest newsletter sections based on topics diff --git a/.opencode/skills/Utilities/Parser/Workflows/CollisionDetection.md b/.opencode/skills/Utilities/Parser/Workflows/CollisionDetection.md new file mode 100755 index 00000000..acf25f97 --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Workflows/CollisionDetection.md @@ -0,0 +1,291 @@ +# Entity Collision Detection Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the CollisionDetection workflow in the Parser skill to prevent duplicates"}' \ + > /dev/null 2>&1 & +``` + +Running the **CollisionDetection** workflow in the **Parser** skill to prevent duplicates... + +## Purpose +Prevent duplicate entities across parsed content by maintaining a global entity index with GUIDs for people, companies, links, and sources. + +## Entity Index Location +`~/.opencode/skills/parser/entity-index.json` + +## How It Works + +### 1. Entity Canonical Identifiers + +Each entity type has a canonical identifier used for collision detection: + +- **People**: Normalized name (lowercase, trimmed) + - `"John Smith"` → `"john smith"` + - `"Andrej Karpathy"` → `"andrej karpathy"` + +- **Companies**: Normalized domain (lowercase, trimmed) + - `"anthropic.com"` → `"anthropic.com"` + - `"openai.com"` → `"openai.com"` + - If no domain: normalized name + +- **Links**: Normalized URL (lowercase, trimmed, no trailing slash) + - `"https://example.com/"` → `"https://example.com"` + - `"HTTPS://SITE.COM/Path"` → `"https://site.com/path"` + +- **Sources**: Normalized URL if available, otherwise normalized author + publication + - With URL: `"https://arxiv.org/abs/123"` → `"https://arxiv.org/abs/123"` + - Without URL: `"Claude Shannon"` + `"Information Theory"` → `"claude shannon|information theory"` + +### 2. Collision Detection Process + +When parsing new content: + +#### Step 1: Load Entity Index +```python +import json + +with open('entity-index.json', 'r') as f: + entity_index = json.load(f) +``` + +#### Step 2: Check Each Extracted Entity + +For each person/company/link/source extracted: + +1. Compute canonical identifier +2. Check if canonical ID exists in index +3. If exists: **Reuse existing GUID** +4. If not exists: **Generate new GUID and add to index** + +#### Step 3: Update Index + +After processing all entities, update the entity index file with new entries. + +### 3. Entity Index Structure + +```json +{ + "version": "1.0.0", + "last_updated": "2025-11-14T18:00:00Z", + "people": { + "john smith": { + "id": "uuid-here", + "name": "John Smith", + "first_seen": "2025-11-14T17:30:10Z", + "occurrences": 2, + "content_ids": ["content-uuid-1", "content-uuid-2"] + } + }, + "companies": { + "anthropic.com": { + "id": "uuid-here", + "name": "Anthropic", + "domain": "anthropic.com", + "first_seen": "2025-11-14T17:30:10Z", + "occurrences": 1, + "content_ids": ["content-uuid-1"] + } + }, + "links": { + "https://example.com/blog/post": { + "id": "uuid-here", + "url": "https://example.com/blog/post", + "first_seen": "2025-11-14T17:30:10Z", + "occurrences": 1, + "content_ids": ["content-uuid-1"] + } + }, + "sources": { + "https://arxiv.org/abs/123": { + "id": "uuid-here", + "url": "https://arxiv.org/abs/123", + "author": "Claude Shannon", + "publication": "Information Theory", + "first_seen": "2025-11-14T17:30:10Z", + "occurrences": 1, + "content_ids": ["content-uuid-1"] + } + } +} +``` + +### 4. Benefits + +**Deduplication:** +- Avoid creating duplicate entities across content +- Maintain single source of truth for each entity + +**Knowledge Graph:** +- Track entity occurrences across content +- Build relationships between content items +- Enable "who else mentioned this person?" queries + +**Skip Already Parsed Content:** +- Check if URL already exists in links index +- Skip re-parsing if content GUID already exists + +**Collision Detection:** +- Detect when same person/company appears in multiple pieces +- Merge information over time + +## Implementation Examples + +### Python Implementation + +```python +import json +import uuid +from datetime import datetime + +def normalize_name(name): + """Normalize person/company name for canonical ID""" + return name.lower().strip() + +def normalize_url(url): + """Normalize URL for canonical ID""" + return url.lower().strip().rstrip('/') + +def get_or_create_person(person_data, entity_index, content_id): + """Get existing GUID or create new one for person""" + canonical_id = normalize_name(person_data['name']) + + if canonical_id in entity_index['people']: + # Entity exists - reuse GUID + existing = entity_index['people'][canonical_id] + existing['occurrences'] += 1 + existing['content_ids'].append(content_id) + return existing['id'] + else: + # New entity - create GUID + person_id = str(uuid.uuid4()) + entity_index['people'][canonical_id] = { + 'id': person_id, + 'name': person_data['name'], + 'first_seen': datetime.utcnow().isoformat() + 'Z', + 'occurrences': 1, + 'content_ids': [content_id] + } + return person_id + +# Similar functions for companies, links, sources... +``` + +### TypeScript Implementation + +```typescript +import { v4 as uuidv4 } from 'uuid'; + +interface EntityIndex { + version: string; + last_updated: string; + people: Record; + companies: Record; + links: Record; + sources: Record; +} + +function normalizeName(name: string): string { + return name.toLowerCase().trim(); +} + +function normalizeUrl(url: string): string { + return url.toLowerCase().trim().replace(/\/$/, ''); +} + +function getOrCreatePerson( + personData: { name: string; /* ... */ }, + entityIndex: EntityIndex, + contentId: string +): string { + const canonicalId = normalizeName(personData.name); + + if (entityIndex.people[canonicalId]) { + // Reuse existing GUID + const existing = entityIndex.people[canonicalId]; + existing.occurrences++; + existing.content_ids.push(contentId); + return existing.id; + } else { + // Create new GUID + const personId = uuidv4(); + entityIndex.people[canonicalId] = { + id: personId, + name: personData.name, + first_seen: new Date().toISOString(), + occurrences: 1, + content_ids: [contentId] + }; + return personId; + } +} +``` + +## Content Duplication Detection + +Before parsing a URL, check if it's already in the index: + +```python +def is_content_already_parsed(url, entity_index): + """Check if this URL has already been parsed""" + canonical_url = normalize_url(url) + return canonical_url in entity_index['links'] + +def get_existing_content_id(url, entity_index): + """Get content ID for already-parsed URL""" + canonical_url = normalize_url(url) + if canonical_url in entity_index['links']: + link_data = entity_index['links'][canonical_url] + if link_data['content_ids']: + return link_data['content_ids'][0] # Original content that included this link + return None +``` + +## Workflow Integration + +### Before Parsing +1. Load entity index +2. Check if source URL already exists in links index +3. If exists and is source link: skip parsing, return existing content ID +4. If not exists or is reference link: proceed with parsing + +### During Entity Extraction +1. For each extracted person/company/link/source +2. Compute canonical identifier +3. Call `get_or_create_entity()` to get GUID +4. Assign GUID to entity in output JSON + +### After Parsing +1. Save updated entity index +2. Update last_updated timestamp +3. Write to disk atomically (write to temp file, then rename) + +## Schema Version + +Current schema version: **1.0.0** + +All entities must include GUID as required field (updated schema: `content-schema.json`). + +## Maintenance + +### Periodic Cleanup +- Remove entities with 0 occurrences (orphaned) +- Merge duplicate entries with similar canonical IDs +- Update stale metadata + +### Backup +Entity index should be backed up regularly: +```bash +cp entity-index.json entity-index.backup.json +``` + +## Future Enhancements + +1. **Fuzzy Matching**: Detect similar names (e.g., "Jon Smith" vs "John Smith") +2. **Alias Resolution**: Track multiple names for same entity +3. **Entity Merging**: UI to merge duplicate entities +4. **Conflict Resolution**: Handle cases where same name refers to different people +5. **Full-Text Search**: Enable searching across all indexed entities diff --git a/.opencode/skills/Utilities/Parser/Workflows/DetectContentType.md b/.opencode/skills/Utilities/Parser/Workflows/DetectContentType.md new file mode 100755 index 00000000..baf9f0e4 --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Workflows/DetectContentType.md @@ -0,0 +1,339 @@ +# Content Type Detection Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the DetectContentType workflow in the Parser skill to detect content"}' \ + > /dev/null 2>&1 & +``` + +Running the **DetectContentType** workflow in the **Parser** skill to detect content... + +**Purpose:** Detect content type from URL to route to appropriate extraction workflow + +**When to Use:** Before parsing any URL, determine what type of content it is + +--- + +## Detection Logic + +### 1. Domain-Based Detection (Highest Priority) + +**YouTube Videos:** +``` +Domain matches: +- youtube.com +- youtu.be +- m.youtube.com + +Path patterns: +- /watch?v=* +- /v/* +- /embed/* +- youtu.be/* (short links) + +→ Type: "video" +→ Route to: Workflows/ExtractYoutube.md +``` + +**Twitter/X Threads:** +``` +Domain matches: +- twitter.com +- x.com +- mobile.twitter.com + +Path patterns: +- /*/status/* (individual tweet or thread start) +- /i/web/status/* (web interface) + +→ Type: "tweet_thread" +→ Route to: Workflows/ExtractTwitter.md +``` + +**Newsletter Platforms:** +``` +Domain matches: +- *.substack.com +- *.beehiiv.com +- *.convertkit.com +- *.ghost.io +- buttondown.email + +→ Type: "newsletter" +→ Route to: Workflows/ExtractNewsletter.md +``` + +**ArXiv Papers:** +``` +Domain matches: +- arxiv.org + +Path patterns: +- /pdf/* (direct PDF link) +- /abs/* (abstract page, extract PDF link) + +→ Type: "pdf" +→ Route to: Workflows/ExtractPdf.md +``` + +--- + +### 2. Extension-Based Detection + +**PDF Documents:** +``` +URL ends with: +- .pdf + +Content-Type header: +- application/pdf + +→ Type: "pdf" +→ Route to: Workflows/ExtractPdf.md +``` + +--- + +### 3. Content-Type Header Detection + +**After fetching URL, check Content-Type:** + +```typescript +If Content-Type === "application/pdf": + → Type: "pdf" + → Route to: Workflows/ExtractPdf.md + +If Content-Type === "video/mp4" or similar: + → Type: "video" + → Extract if possible, otherwise flag as unsupported + +If Content-Type === "text/html": + → Continue to HTML analysis (step 4) +``` + +--- + +### 4. HTML Metadata Analysis + +**For HTML content, check OpenGraph and meta tags:** + +**Video Detection:** +```html + + + +→ Check if YouTube embed → Route to YouTube workflow +→ Otherwise: Extract video URL, attempt download +``` + +**Article Detection:** +```html + + + + +→ Type: "article" +→ Route to: Workflows/ExtractArticle.md +``` + +**Podcast Detection:** +```html + (often used for podcasts) +Links to .mp3, .m4a audio files + +→ Type: "podcast" +→ Note: Transcript extraction may not be available +→ Route to: Workflows/ExtractArticle.md (extract show notes) +``` + +--- + +### 5. Default Fallback + +**If no specific type detected:** +``` +→ Type: "generic" +→ Route to: Workflows/ExtractArticle.md +→ Flag: Lower confidence score +→ Extract whatever content is available +``` + +--- + +## Detection Algorithm + +```typescript +function detectContentType(url: string): ContentType { + // 1. Parse URL + const parsedUrl = new URL(url); + const domain = parsedUrl.hostname; + const path = parsedUrl.pathname; + const extension = path.split('.').pop()?.toLowerCase(); + + // 2. Domain-based detection (highest priority) + if (domain.includes('youtube.com') || domain.includes('youtu.be')) { + return 'video'; + } + + if (domain.includes('twitter.com') || domain.includes('x.com')) { + if (path.includes('/status/')) { + return 'tweet_thread'; + } + } + + if (domain.includes('substack.com') || + domain.includes('beehiiv.com') || + domain.includes('convertkit.com') || + domain.includes('ghost.io') || + domain.includes('buttondown.email')) { + return 'newsletter'; + } + + if (domain.includes('arxiv.org')) { + return 'pdf'; + } + + // 3. Extension-based detection + if (extension === 'pdf') { + return 'pdf'; + } + + // 4. Fetch and check Content-Type + const response = await fetch(url, { method: 'HEAD' }); + const contentType = response.headers.get('content-type'); + + if (contentType?.includes('application/pdf')) { + return 'pdf'; + } + + if (contentType?.includes('text/html')) { + // 5. HTML metadata analysis + const html = await fetch(url).then(r => r.text()); + const ogType = extractMetaTag(html, 'og:type'); + + if (ogType === 'video') { + return 'video'; + } + + if (ogType === 'article') { + return 'article'; + } + + if (ogType === 'music.song') { + return 'podcast'; + } + } + + // 6. Default fallback + return 'generic'; +} +``` + +--- + +## Routing Map + +**Content Type → Extraction Workflow** + +``` +video → Workflows/ExtractYoutube.md +tweet_thread → Workflows/ExtractTwitter.md +newsletter → Workflows/ExtractNewsletter.md +pdf → Workflows/ExtractPdf.md +article → Workflows/ExtractArticle.md +podcast → Workflows/ExtractArticle.md (extract show notes) +generic → Workflows/ExtractArticle.md (fallback) +``` + +--- + +## Examples + +**Example 1: YouTube Video** +``` +Input: https://youtube.com/watch?v=abc123 + +Detection: +1. Domain = "youtube.com" → MATCH +2. Path = "/watch?v=abc123" → MATCH pattern +3. Type = "video" +4. Route to: Workflows/ExtractYoutube.md +``` + +**Example 2: Substack Newsletter** +``` +Input: https://newsletter.substack.com/p/weekly-update + +Detection: +1. Domain = "newsletter.substack.com" → MATCH +2. Type = "newsletter" +3. Route to: Workflows/ExtractNewsletter.md +``` + +**Example 3: ArXiv PDF** +``` +Input: https://arxiv.org/pdf/2401.12345.pdf + +Detection: +1. Domain = "arxiv.org" → MATCH +2. Path includes "/pdf/" → MATCH +3. Extension = ".pdf" → MATCH +4. Type = "pdf" +5. Route to: Workflows/ExtractPdf.md +``` + +**Example 4: Generic Blog Post** +``` +Input: https://randomblog.com/post/interesting-article + +Detection: +1. Domain = "randomblog.com" → No match +2. Extension = None +3. Fetch Content-Type = "text/html" +4. Check OpenGraph: og:type = "article" → MATCH +5. Type = "article" +6. Route to: Workflows/ExtractArticle.md +``` + +**Example 5: Unknown Content** +``` +Input: https://unknown-site.com/page + +Detection: +1. Domain → No match +2. Extension → No match +3. Content-Type = "text/html" +4. OpenGraph → No article type found +5. Type = "generic" (fallback) +6. Route to: Workflows/ExtractArticle.md +7. Flag: Lower confidence score +``` + +--- + +## Error Handling + +**If URL is inaccessible:** +- Return error with 404/403/500 status +- Log to extraction_metadata.warnings +- Attempt to use archive.org cached version +- If archive fails, return partial data with low confidence + +**If Content-Type is ambiguous:** +- Default to "generic" type +- Route to article extraction (most versatile) +- Set confidence_score to 0.5 or lower +- Log ambiguity in warnings + +**If redirect chain is too long:** +- Follow up to 5 redirects +- Use final destination URL for detection +- Log redirect chain in warnings +- Update canonical_url in metadata + +--- + +**This workflow ensures accurate content type detection for optimal extraction routing.** diff --git a/.opencode/skills/Utilities/Parser/Workflows/ExtractArticle.md b/.opencode/skills/Utilities/Parser/Workflows/ExtractArticle.md new file mode 100755 index 00000000..c65bd402 --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Workflows/ExtractArticle.md @@ -0,0 +1,474 @@ +# Article Extraction Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the ExtractArticle workflow in the Parser skill to parse articles"}' \ + > /dev/null 2>&1 & +``` + +Running the **ExtractArticle** workflow in the **Parser** skill to parse articles... + +**Purpose:** Extract content, metadata, and entities from web articles and generic web pages + +**When to Use:** Content type detected as "article" or "generic" web content + +--- + +## Extraction Steps + +### 1. Content Scraping (Gemini Researcher) + +**Use GeminiResearcher agent for full content extraction:** + +Launch Gemini Researcher with 2M token context: +``` +Prompt: "Extract the full article content from this URL including all text, metadata, and structure. Preserve formatting and identify main content vs sidebar/ads." +URL: [article URL] +``` + +**Extract:** +- Main article body (clean text) +- Headlines and subheadlines +- Pull quotes and callouts +- Image captions +- Author byline +- Publication date and time +- Publication name +- Article category/section + +**Content cleaning:** +- Remove ads and promotional content +- Remove navigation menus +- Remove cookie banners +- Remove social share buttons +- Preserve article text, quotes, and inline links + +--- + +### 2. HTML Metadata Extraction + +**Parse HTML for structured metadata:** + +**OpenGraph tags:** +```html + + + + + + +``` + +**Twitter Card tags:** +```html + + + + +``` + +**Article-specific tags:** +```html + + + + + +``` + +**JSON-LD structured data:** +```json +{ + "@context": "https://schema.org", + "@type": "Article", + "headline": "...", + "author": { + "@type": "Person", + "name": "Author Name", + "url": "..." + }, + "datePublished": "2024-01-15T10:00:00Z", + "publisher": { + "@type": "Organization", + "name": "Publication Name" + } +} +``` + +--- + +### 3. Author Information Extraction + +**Extract author details:** + +**From metadata:** +- Author name (meta tags, byline, JSON-LD) +- Author bio (if available on page) +- Author photo URL +- Author social links (Twitter, LinkedIn) + +**Author page scraping (if linked):** +- Full bio +- Email address (if public) +- Website +- Social media profiles +- Other articles by author + +**Add to people array:** +```json +{ + "name": "Author Name", + "role": "author", + "title": "Senior Writer at Publication", + "company": "Publication Name", + "social": { + "twitter": "@authorhandle", + "linkedin": "https://linkedin.com/in/authorname", + "email": "author@publication.com", + "website": "https://authorwebsite.com" + }, + "context": "Article author, writes about AI and technology", + "importance": "primary" +} +``` + +--- + +### 4. Link Extraction and Classification + +**Extract all links from article body:** + +**For each link:** +- URL +- Anchor text (link title) +- Surrounding context (sentence/paragraph) +- Position in article (beginning, middle, end) + +**Classify link types:** +- **reference**: Citations to other articles/sources +- **source**: Primary sources (research papers, official docs) +- **related**: Related articles on same site +- **tool**: Links to tools, products, services +- **research**: Academic papers, studies +- **product**: Commercial products mentioned +- **social**: Social media profiles +- **other**: Miscellaneous + +**Example:** +```json +{ + "url": "https://arxiv.org/abs/2401.12345", + "domain": "arxiv.org", + "title": "Attention Is All You Need", + "description": "Research paper on transformer architecture", + "link_type": "research", + "context": "Article cites this paper when discussing transformer models", + "position": "middle" +} +``` + +--- + +### 5. Gemini Entity Extraction + +**Send article content to Gemini:** + +**Prompt:** Use `prompts/entity-extraction.md` + +**Extract:** + +**People mentioned:** +- Names of people discussed in article +- Roles and titles +- Companies/organizations they're affiliated with +- Why they're mentioned + +**Companies/organizations:** +- All companies mentioned +- Industry classifications +- How they're discussed (subject, competitor, example, etc.) +- Sentiment (positive, neutral, negative) + +**Products/tools:** +- Software, hardware, services mentioned +- Associated companies +- Context of mention + +**Example extractions:** +```json +// Person +{ + "name": "Sam Altman", + "role": "subject", + "title": "CEO of OpenAI", + "company": "OpenAI", + "social": { + "twitter": "@sama", + "linkedin": null, + "email": null, + "website": null + }, + "context": "Quoted discussing GPT-4 capabilities and future plans", + "importance": "primary" +} + +// Company +{ + "name": "OpenAI", + "domain": "openai.com", + "industry": "AI", + "context": "Main subject of article - discussed new model release", + "mentioned_as": "subject", + "sentiment": "positive" +} +``` + +--- + +### 6. Summarization + +**Use Gemini for multi-level summaries:** + +**Prompt:** Use `prompts/summarization.md` + +**Short summary (1-2 sentences):** +``` +OpenAI announces GPT-4, a new large language model with significantly improved reasoning capabilities and multimodal input support. +``` + +**Medium summary (paragraph):** +``` +OpenAI has released GPT-4, the latest iteration of its large language model, featuring substantial improvements in reasoning, problem-solving, and the ability to process both text and images. The model demonstrates better performance on academic benchmarks, reduced hallucinations, and enhanced safety guardrails. GPT-4 is being rolled out to ChatGPT Plus subscribers and enterprise customers, with API access coming soon. +``` + +**Long summary (multiple paragraphs):** +``` +[Comprehensive summary covering: main announcement, key technical improvements, benchmark results, safety considerations, availability timeline, industry reactions, implications for AI development, notable quotes, and future roadmap] +``` + +**Key excerpts:** +- Pull quotes from sources +- Notable statistics or findings +- Surprising insights + +--- + +### 7. Topic Classification + +**Analyze content for categorization:** + +**Prompt:** Use `prompts/topic-classification.md` + +**Determine:** +- Primary category (AI, security, technology, business, science, etc.) +- Secondary categories +- Specific tags +- Keywords +- Broader themes +- Which UL newsletter sections this fits + +**Example:** +```json +{ + "primary_category": "AI", + "secondary_categories": ["technology", "business"], + "tags": [ + "gpt-4", + "openai", + "large-language-models", + "multimodal-ai", + "chatgpt" + ], + "keywords": [ + "GPT-4", + "OpenAI", + "language model", + "reasoning capabilities", + "multimodal" + ], + "themes": [ + "AI model capabilities expanding", + "Multimodal AI emerging", + "AI safety and alignment" + ], + "newsletter_sections": ["AI", "Headlines", "Tools"] +} +``` + +--- + +### 8. Source Extraction + +**Identify cited sources in article:** + +**Look for:** +- Research papers cited +- Previous news articles referenced +- Official statements/press releases +- Data sources (reports, studies) +- Expert quotes with attribution + +**For each source:** +```json +{ + "publication": "Nature", + "author": "Smith et al.", + "url": "https://nature.com/articles/paper123", + "published_date": "2023-12-01T00:00:00Z", + "source_type": "research_paper" +} +``` + +--- + +### 9. Content Analysis + +**Analyze article for:** + +**Sentiment:** +- Positive (optimistic, favorable) +- Neutral (factual, balanced) +- Negative (critical, cautionary) +- Mixed (balanced pros/cons) + +**Importance score (1-10):** +- Major announcement/breakthrough: 8-10 +- Significant development: 6-8 +- Incremental progress: 4-6 +- Minor update: 1-3 + +**Novelty score (1-10):** +- Original research/exclusive: 9-10 +- First coverage of new topic: 7-9 +- New angle on known topic: 5-7 +- Recap/summary: 1-4 + +**Controversy score (1-10):** +- Highly debatable/polarizing: 8-10 +- Some controversy: 5-7 +- Generally accepted: 1-4 + +**Relevance to audience:** +- Security professionals +- AI researchers +- Technologists +- Executives +- Entrepreneurs +- General tech audience + +**Trending potential:** +- High: Viral topic, breaking news +- Medium: Important but niche +- Low: Evergreen content + +**Key insights:** +- Main takeaways (3-5 bullet points) +- Novel information +- Action items or implications + +--- + +### 10. Schema Population + +**Populate content section:** +```json +{ + "content": { + "id": "uuid-generated", + "type": "article", + "title": "OpenAI Announces GPT-4", + "summary": { + "short": "...", + "medium": "...", + "long": "..." + }, + "content": { + "full_text": "[Complete article text]", + "transcript": null, + "excerpts": [ + "\"GPT-4 is our most capable model yet\" - Sam Altman", + "Key stat about performance" + ] + }, + "metadata": { + "source_url": "https://openai.com/blog/gpt-4", + "canonical_url": "https://openai.com/blog/gpt-4", + "published_date": "2024-01-15T10:00:00Z", + "accessed_date": "2024-01-16T14:30:00Z", + "language": "en", + "word_count": 1500, + "read_time_minutes": 6, + "author_platform": "blog" + } + } +} +``` + +**Set confidence score:** +- Full extraction with metadata: 0.85-0.95 +- Good content, missing some metadata: 0.7-0.84 +- Partial extraction: 0.5-0.69 +- Minimal/generic extraction: 0.3-0.49 + +--- + +## Error Handling + +**Paywall content:** +- Extract what's visible (headline, summary) +- Note in warnings: "Content behind paywall" +- Lower confidence score +- Attempt archive.org cached version + +**JavaScript-heavy sites:** +- Use headless browser if needed +- Wait for dynamic content to load +- Extract rendered HTML +- May increase processing time + +**403/401 errors:** +- Log error in warnings +- Attempt user-agent spoofing +- Try archive.org +- Return partial metadata + +**Rate limiting:** +- Respect robots.txt +- Back off if rate limited +- Retry with exponential backoff +- Log in warnings + +--- + +## Output Example + +**Successful extraction:** +``` +✅ Extracted article +📰 Title: OpenAI Announces GPT-4 +✍️ Author: Sam Altman +📅 Published: 2024-01-15 +📝 Content: 1,500 words (~6 min read) +👥 People: 3 +🏢 Companies: 5 +🔗 Links: 12 +📚 Sources: 4 +🎯 Confidence: 0.91 +``` + +**Partial extraction:** +``` +⚠️ Extracted article (partial - paywall detected) +📰 Title: Premium Content Article +📝 Content: 300 words visible (full text unavailable) +🔗 Links: 3 (from visible portion) +🎯 Confidence: 0.52 +⚠️ Warning: Content behind paywall, extracted preview only +``` + +--- + +**This workflow handles comprehensive article extraction with Gemini Researcher.** diff --git a/.opencode/skills/Utilities/Parser/Workflows/ExtractBrowserExtension.md b/.opencode/skills/Utilities/Parser/Workflows/ExtractBrowserExtension.md new file mode 100644 index 00000000..819b6fff --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Workflows/ExtractBrowserExtension.md @@ -0,0 +1,335 @@ +# Browser Extension Security Analysis Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the ExtractBrowserExtension workflow in the Parser skill to analyze extensions"}' \ + > /dev/null 2>&1 & +``` + +Running the **ExtractBrowserExtension** workflow in the **Parser** skill to analyze browser extensions... + +**Purpose:** Analyze browser extensions for security risks, clone detection, and malicious patterns + +**When to Use:** +- Security assessment of unknown extensions +- Incident response involving browser-based malware +- Vetting extensions before enterprise deployment +- Analyzing suspicious "Click Fix" or "Crash Fix" style attacks + +--- + +## Extraction Steps + +### 1. Extension Acquisition + +**For Chrome Web Store extensions:** +```bash +# Get CRX from Chrome Web Store +# Extension ID is the last part of the store URL +EXTENSION_ID="abcdefghijklmnop" +CRX_URL="https://clients2.google.com/service/update2/crx?response=redirect&prodversion=49.0&acceptformat=crx3&x=id%3D${EXTENSION_ID}%26installsource%3Dondemand%26uc" + +curl -L -o extension.crx "$CRX_URL" +``` + +**For local/sideloaded extensions:** +```bash +# Chrome extensions directory +~/.config/google-chrome/Default/Extensions/ + +# Extract from CRX (it's a ZIP with extra header) +unzip extension.crx -d extension_unpacked/ +``` + +--- + +### 2. Manifest Analysis + +**Parse manifest.json for red flags:** + +```json +{ + "manifest_version": 3, + "permissions": ["", "clipboardWrite", "clipboardRead"], + "host_permissions": ["*://*/*"], + "background": { + "service_worker": "background.js" + }, + "content_scripts": [{ + "matches": [""], + "js": ["content.js"] + }] +} +``` + +**High-Risk Permission Indicators:** + +| Permission | Risk Level | Why | +|------------|------------|-----| +| `` | HIGH | Access to all websites | +| `clipboardWrite/Read` | HIGH | Clipboard manipulation (Click Fix) | +| `tabs` | MEDIUM | Tab enumeration | +| `webRequest` | HIGH | Intercept/modify traffic | +| `nativeMessaging` | HIGH | Communicate with native apps | +| `management` | HIGH | Control other extensions | +| `debugger` | CRITICAL | Debug other tabs | + +--- + +### 3. Clone Detection + +**Compare against known legitimate extensions:** + +```bash +# Download known-good version +# For example, uBlock Origin Light +git clone https://github.com/AykutSarac/ublock-origin-lite.git legitimate/ + +# Diff against suspect +diff -r suspect_extension/ legitimate/ + +# Key indicators: +# - Brand name replacements ("uBlock" -> "NextShield") +# - Broken URLs from find/replace +# - Additional JS files not in original +# - Modified permissions in manifest +``` + +**Clone Detection Checklist:** +- [ ] Compare file structure to claimed origin extension +- [ ] Check for find/replace artifacts in comments/URLs +- [ ] Verify manifest permissions match original +- [ ] Look for additional obfuscated files +- [ ] Check author/publisher information + +--- + +### 4. Alarms API Analysis + +**Detect delayed execution (evasion technique):** + +```bash +# Search for Chrome Alarms usage +grep -r "chrome.alarms" extension_unpacked/ + +# Red flag patterns: +# - delayInMinutes > 30 (decoupling from install) +# - periodInMinutes with network callbacks +``` + +**Example malicious pattern:** +```javascript +// SUSPICIOUS: 60-minute delay then 10-minute intervals +chrome.alarms.create('update', { + delayInMinutes: 60, + periodInMinutes: 10 +}); + +chrome.alarms.onAlarm.addListener((alarm) => { + // Malicious activity starts HERE, long after install + checkCommand(); +}); +``` + +--- + +### 5. Background Script Analysis + +**Analyze service worker/background scripts:** + +```bash +# Deobfuscate if needed +npm install -g js-beautify +js-beautify background.js > background_pretty.js + +# Search for indicators +grep -E "(clipboard|fetch|XMLHttpRequest|eval|atob|fromCharCode)" background_pretty.js +``` + +**Malicious Indicators:** +- Clipboard manipulation (`navigator.clipboard.writeText`) +- Obfuscated strings (`atob`, `String.fromCharCode`) +- External C2 connections (`fetch` to unknown domains) +- `eval()` or `new Function()` execution +- DOM manipulation on all pages + +--- + +### 6. Network Analysis + +**Identify C2 domains and external connections:** + +```bash +# Extract all URLs/domains from extension +grep -rhoE 'https?://[^"'\''>\s]+' extension_unpacked/ | sort -u + +# Check domains against threat intel +# - VirusTotal +# - URLhaus +# - AbuseIPDB +``` + +**C2 Indicators:** +- Newly registered domains (< 30 days) +- DGA-looking domains +- IP addresses instead of domains +- Base64/encoded URLs +- POST requests with system info + +--- + +### 7. Content Script Analysis + +**Check what runs on web pages:** + +```bash +# Find content scripts +grep -l "content_scripts" manifest.json +cat manifest.json | jq '.content_scripts' + +# Analyze injection patterns +grep -E "(keylogger|password|credit|ssn|inject)" content.js +``` + +**Risk Patterns:** +- Form field interception +- Keyboard event listeners +- DOM modification +- Screenshot capabilities +- Cookie theft + +--- + +### 8. Fingerprinting Detection + +**Check for environment fingerprinting:** + +```bash +# Search for system enumeration +grep -E "(navigator|screen|platform|userAgent|plugins)" extension_unpacked/*.js +``` + +**Targeting Indicators (from Crash Fix):** +```javascript +// Corporate vs consumer targeting +(Get-WmiObject -Class Win32_ComputerSystem).PartOfDomain +[System.DirectoryServices.ActiveDirectory.Domain]::GetCurrentDomain() + +// Malware checks these to serve different payloads: +// - Domain-joined = corporate = high-value target +// - Standalone = consumer = lower priority +``` + +--- + +### 9. Schema Population + +**Output security analysis:** + +```json +{ + "extension": { + "id": "extension-id", + "name": "Claimed Name", + "version": "1.0.0", + "analysis_date": "2026-01-23" + }, + "security_assessment": { + "risk_level": "HIGH|MEDIUM|LOW|CRITICAL", + "is_clone": true, + "clone_of": "uBlock Origin Light", + "confidence": 0.95 + }, + "indicators": { + "permissions": { + "high_risk": ["", "clipboardWrite"], + "medium_risk": ["tabs"] + }, + "evasion_techniques": [ + "Chrome Alarms delay (60 min)", + "Obfuscated code" + ], + "c2_domains": [ + "malicious-domain.com" + ], + "lotl_binaries": [ + "finger.exe" + ] + }, + "recommendations": [ + "Remove extension immediately", + "Check clipboard for malicious content", + "Monitor for persistence mechanisms" + ] +} +``` + +--- + +## Error Handling + +**Extension not available:** +- Log error: "Extension not found or removed from store" +- Check Wayback Machine for cached version +- Note: Removal may indicate detection + +**Obfuscated code:** +- Attempt deobfuscation with js-beautify +- Use AST analysis for complex obfuscation +- Lower confidence score if analysis incomplete + +**Manifest V3 restrictions:** +- Service workers instead of background pages +- Note: Some malicious capabilities limited in V3 +- Still check for workarounds + +--- + +## Output Example + +**Malicious extension detected:** +``` +CRITICAL: Malicious browser extension detected + +Extension: "NextShield Ad Blocker" v1.2.3 +Clone Of: uBlock Origin Light (95% code match) + +Risk Level: CRITICAL + +Indicators Found: +- Clone of legitimate extension with brand replacement +- Chrome Alarms delay: 60 minutes (evasion) +- Clipboard manipulation capability +- C2 domain: malicious-server[.]com +- Targets domain-joined machines (corporate targeting) + +Attack Chain: "Crash Fix" variant +1. User installs fake extension +2. 60-minute delay before activation +3. Simulates browser crash +4. Injects payload into clipboard +5. Social engineers Win+R, Ctrl+V execution +6. Uses finger.exe for C2 communication + +Recommendations: +1. Remove extension immediately +2. Clear clipboard +3. Check for persistence (scheduled tasks, registry) +4. Review browsing history during delay period +5. Report to Chrome Web Store +``` + +--- + +## Related Workflows + +- **EntityLookup** - For C2 domain investigation +- **LOTLBinaries.md** - For LOTL technique reference + +--- + +**Last Updated:** 2026-01-23 +**Source:** Matt Johansen security research, Huntress "Crash Fix" analysis diff --git a/.opencode/skills/Utilities/Parser/Workflows/ExtractNewsletter.md b/.opencode/skills/Utilities/Parser/Workflows/ExtractNewsletter.md new file mode 100755 index 00000000..70bb7afe --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Workflows/ExtractNewsletter.md @@ -0,0 +1,320 @@ +# Newsletter Extraction Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the ExtractNewsletter workflow in the Parser skill to parse newsletters"}' \ + > /dev/null 2>&1 & +``` + +Running the **ExtractNewsletter** workflow in the **Parser** skill to parse newsletters... + +**Purpose:** Extract structured content from newsletter HTML (Substack, Beehiiv, ConvertKit, etc.) + +**When to Use:** Content type detected as "newsletter" from known newsletter domains + +--- + +## Extraction Steps + +### 1. Newsletter Platform Detection + +**Identify platform:** +- Substack: `*.substack.com` +- Beehiiv: `*.beehiiv.com` +- ConvertKit: `*.convertkit.com` +- Ghost: `*.ghost.io` +- Buttondown: `buttondown.email` + +**Platform-specific parsing:** +- Each platform has different HTML structure +- Adapt selectors based on platform +- Extract platform-specific metadata + +--- + +### 2. Newsletter Metadata Extraction + +**Extract newsletter-level metadata:** +- Newsletter name +- Issue number (if present) +- Publication date +- Author/creator +- Newsletter description +- Subscribe URL +- Social links + +**Example (Substack):** +```html + + + +``` + +--- + +### 3. Section/Story Parsing + +**Parse newsletter structure:** + +**Common sections:** +- Main story/featured content +- News roundup +- Tool recommendations +- Sponsored content +- Upcoming events +- Community highlights + +**For each section:** +- Section title/header +- Section description +- Individual items/stories within section + +**Example structure:** +``` +Newsletter: AI Weekly #42 +├── Main Story: GPT-4 Announced +├── Headlines (5 items) +│ ├── OpenAI releases GPT-4 +│ ├── Google announces Gemini Pro +│ ├── Meta open-sources Llama 3 +│ ├── Anthropic raises $500M +│ └── New AI safety regulations +├── Tools (3 items) +│ ├── LangChain v0.2 +│ ├── AutoGPT updates +│ └── New Hugging Face models +└── Analysis: AI Scaling Laws Revisited +``` + +--- + +### 4. Individual Item Extraction + +**For each newsletter item/story:** + +**Extract:** +- Item title/headline +- Item description/summary +- Author commentary (if separate from description) +- Links (primary link + any supporting links) +- Position in newsletter (section + order) +- Inline images (if relevant) + +**Example item:** +```json +{ + "title": "OpenAI Releases GPT-4", + "summary": "OpenAI's latest model shows significant improvements in reasoning...", + "editorial_note": "This is huge - multimodal capabilities open new possibilities", + "links": [ + { + "url": "https://openai.com/blog/gpt-4", + "title": "Official announcement", + "type": "primary" + }, + { + "url": "https://twitter.com/sama/status/123", + "title": "Sam Altman's tweet", + "type": "reference" + } + ], + "section": "Headlines", + "position": 1 +} +``` + +--- + +### 5. Link Context Extraction + +**For all links in newsletter:** + +**Extract context:** +- What section is the link in? +- What's the surrounding text? +- Is it the primary source or supporting reference? +- Author's commentary on the link + +**Example:** +```json +{ + "url": "https://openai.com/blog/gpt-4", + "domain": "openai.com", + "title": "GPT-4 Announcement", + "description": "Official blog post about GPT-4 release", + "link_type": "source", + "context": "Featured in 'Headlines' section as major AI announcement with positive commentary", + "position": "beginning" +} +``` + +--- + +### 6. Author/Editorial Voice Extraction + +**Identify author's commentary:** + +**Distinguish between:** +- Original source material (quoted/linked) +- Author's commentary/analysis +- Editorial notes +- Personal takes + +**Example:** +``` +Original: "GPT-4 achieves 90th percentile on bar exam" +Author commentary: "This is a significant milestone, though real-world legal work requires more than exam performance" +``` + +**Extract as:** +- `editorial_note` in newsletter_metadata +- Separate excerpts showing author's voice +- Sentiment analysis of author's take + +--- + +### 7. Sponsored Content Detection + +**Identify sponsored/promotional content:** + +**Indicators:** +- "Sponsored by" labels +- "Partner content" labels +- Affiliate disclosure +- Ad blocks + +**Flag sponsored content:** +- Note in `link_type` as "sponsored" +- Lower importance score +- Separate from editorial content + +--- + +### 8. Gemini Analysis + +**Send newsletter content to Gemini:** + +**Extract:** +- All people mentioned across all items +- All companies mentioned +- Topics covered (aggregate across items) +- Overall newsletter themes +- Audience relevance + +**Generate newsletter-level summary:** +- Short: "Weekly AI news covering GPT-4 release, Gemini updates, and new tools" +- Medium: Paragraph summarizing all major items +- Long: Comprehensive summary of entire newsletter + +--- + +### 9. Multi-Item vs Single-Item Mode + +**Decision: Parse as single item or multiple items?** + +**Single-item mode (default):** +- Entire newsletter is one content item +- All links aggregated +- Newsletter-level summary +- Good for: archiving full newsletters + +**Multi-item mode (optional future feature):** +- Each newsletter section/story becomes separate content item +- More granular database entries +- Better for: extracting individual stories for newsletter database + +**For now: Use single-item mode** +- Simpler implementation +- Preserves newsletter structure +- Can split later if needed + +--- + +### 10. Schema Population + +**Populate content section:** +```json +{ + "content": { + "id": "uuid-generated", + "type": "newsletter", + "title": "AI Weekly #42 - GPT-4 and More", + "summary": { + "short": "Weekly AI news covering GPT-4, Gemini, Llama 3, and new tools", + "medium": "This week's newsletter covers major AI announcements including OpenAI's GPT-4 release with multimodal capabilities, Google's Gemini Pro updates, Meta's Llama 3 open-source release, and a roundup of new tools including LangChain v0.2. Also includes analysis on AI scaling laws and regulatory developments.", + "long": "[Comprehensive summary of all sections and items]" + }, + "content": { + "full_text": "[Full newsletter text with all items]", + "transcript": null, + "excerpts": [ + "\"GPT-4 is a game-changer for multimodal AI\" - Author commentary", + "This week saw three major model releases in one week", + "Scaling laws may be hitting diminishing returns" + ] + }, + "metadata": { + "source_url": "https://newsletter.substack.com/p/issue-42", + "canonical_url": "https://newsletter.substack.com/p/issue-42", + "published_date": "2024-01-15T10:00:00Z", + "accessed_date": "2024-01-16T15:30:00Z", + "language": "en", + "word_count": 2000, + "read_time_minutes": 8, + "author_platform": "substack" + } + }, + "newsletter_metadata": { + "issue_number": 42, + "section": null, + "position_in_section": null, + "editorial_note": "Aggregate of author commentary throughout newsletter", + "include_in_newsletter": false, + "scheduled_date": null + } +} +``` + +**Links array includes ALL links from all newsletter items** + +**Topics aggregate all topics mentioned across items** + +--- + +## Error Handling + +**Paywall/subscription required:** +- Extract preview content if available +- Extract metadata from page +- Note in warnings: "Full content requires subscription" +- Confidence: 0.4-0.6 + +**Complex HTML structure:** +- Fall back to generic article extraction +- May lose newsletter-specific structure +- Log warning +- Confidence: 0.6-0.7 + +--- + +## Output Example + +**Successful extraction:** +``` +✅ Extracted newsletter +📧 Title: AI Weekly #42 - GPT-4 and More +📅 Published: 2024-01-15 +📝 Content: 2,000 words (~8 min read) +📰 Issue: #42 +🔗 Links: 18 (across all sections) +👥 People: 7 +🏢 Companies: 12 +🎯 Confidence: 0.89 +``` + +--- + +**This workflow handles comprehensive newsletter extraction with section/item parsing.** diff --git a/.opencode/skills/Utilities/Parser/Workflows/ExtractPdf.md b/.opencode/skills/Utilities/Parser/Workflows/ExtractPdf.md new file mode 100755 index 00000000..c8c187dc --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Workflows/ExtractPdf.md @@ -0,0 +1,346 @@ +# PDF Document Extraction Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the ExtractPdf workflow in the Parser skill to parse PDFs"}' \ + > /dev/null 2>&1 & +``` + +Running the **ExtractPdf** workflow in the **Parser** skill to parse PDFs... + +**Purpose:** Extract text, metadata, and entities from PDF documents (research papers, reports, ebooks) + +**When to Use:** Content type detected as "pdf" from URL extension or Content-Type header + +--- + +## Extraction Steps + +### 1. PDF Download and Text Extraction + +**Download PDF:** +```bash +curl -L -o temp.pdf "[PDF_URL]" +``` + +**Extract text using pdftotext or similar:** +```bash +pdftotext temp.pdf output.txt +# or +python -c "import PyPDF2; ..." +``` + +**Extract:** +- Full text content +- Preserve structure (sections, paragraphs) +- Extract tables if possible +- Get page count +- Calculate word count + +**Handle encryption/protection:** +- If password-protected, log warning and skip +- If copy-protected, attempt extraction anyway +- Log any extraction issues in warnings + +--- + +### 2. PDF Metadata Extraction + +**Extract PDF metadata:** +```bash +pdfinfo temp.pdf +``` + +**Metadata fields:** +- Title +- Author(s) +- Subject/description +- Keywords +- Creation date +- Modification date +- Producer (PDF creator software) +- Page count + +**Example:** +``` +Title: Attention Is All You Need +Author: Vaswani, Ashish; Shazeer, Noam; et al. +Subject: Transformer architecture for sequence modeling +Keywords: neural networks, attention mechanism, transformers +CreationDate: 2017-06-12 +Pages: 15 +``` + +--- + +### 3. Research Paper Detection + +**Detect if PDF is research paper:** + +**Indicators:** +- ArXiv URL pattern +- Abstract section present +- References/bibliography section +- DOI present +- Academic author affiliations +- Published in conference/journal + +**If research paper:** +- Extract abstract separately +- Extract bibliography/citations +- Identify authors and affiliations +- Extract publication venue +- Get DOI if available + +--- + +### 4. Gemini Analysis + +**Send full text to Gemini for entity extraction:** + +**Prompt:** Use `prompts/entity-extraction.md` + +**Extract:** +- Authors (all listed authors) +- Organizations/institutions (author affiliations) +- Companies mentioned in paper +- People cited in references +- Technical concepts and terminology + +**Example:** +```json +// Author +{ + "name": "Ashish Vaswani", + "role": "author", + "title": "Research Scientist", + "company": "Google Brain", + "social": { + "twitter": null, + "linkedin": null, + "email": null, // often in papers + "website": null + }, + "context": "Lead author of transformer architecture paper", + "importance": "primary" +} +``` + +--- + +### 5. Summarization + +**Use Gemini for summaries:** + +**Short (1-2 sentences):** +``` +Introduces the Transformer architecture based solely on attention mechanisms, eliminating recurrence and convolutions for sequence modeling tasks. +``` + +**Medium (paragraph):** +``` +Presents the Transformer, a novel neural network architecture that relies entirely on self-attention mechanisms rather than recurrent or convolutional layers. The model achieves state-of-the-art results on machine translation tasks while being more parallelizable and requiring less training time than previous architectures. Key innovation is the multi-head attention mechanism that allows the model to attend to information from different representation subspaces. +``` + +**Long (comprehensive):** +``` +[Full summary including: motivation, methodology, results, key innovations, comparisons to prior work, implications, future directions] +``` + +**Key excerpts:** +- Abstract (full text) +- Key findings from results section +- Notable quotes or claims +- Important equations or algorithms (described) + +--- + +### 6. Citation Extraction + +**Extract references/bibliography:** + +**Parse references section:** +- Each cited work +- Authors of cited works +- Publication details +- DOIs if available + +**Add to sources array:** +```json +{ + "publication": "Neural Information Processing Systems (NeurIPS)", + "author": "Sutskever et al.", + "url": "https://arxiv.org/abs/1409.3215", + "published_date": "2014-09-02T00:00:00Z", + "source_type": "research_paper" +} +``` + +--- + +### 7. Topic Classification + +**For research papers:** +- Primary category: Extract from ArXiv category if available +- Tags: Technical terms, methods, applications +- Keywords: From PDF metadata + extracted concepts +- Themes: Main contributions and innovations + +**For reports/ebooks:** +- Analyze table of contents +- Extract chapter titles +- Identify main topics covered + +**Example (research paper):** +```json +{ + "primary_category": "AI", + "secondary_categories": ["research", "machine-learning"], + "tags": [ + "transformers", + "attention-mechanism", + "neural-networks", + "sequence-modeling", + "nlp" + ], + "keywords": [ + "attention", + "transformer", + "self-attention", + "multi-head attention", + "encoder-decoder" + ], + "themes": [ + "Attention-only architecture", + "Parallelizable sequence modeling", + "State-of-the-art machine translation" + ], + "newsletter_sections": ["AI", "Research"] +} +``` + +--- + +### 8. Content Analysis + +**Research papers:** +- Importance: Based on citation count, venue, novelty +- Novelty: Groundbreaking (9-10), incremental (4-6), survey (1-3) +- Controversy: Assess if methods/claims are debated + +**Reports/ebooks:** +- Importance: Based on author authority, publisher, topic relevance +- Novelty: Original research vs compilation +- Controversy: Assess if conclusions are disputed + +**Relevance to audience:** +- AI researchers +- Security professionals +- Technologists +- Executives + +--- + +### 9. Schema Population + +**Populate content section:** +```json +{ + "content": { + "id": "uuid-generated", + "type": "pdf", + "title": "Attention Is All You Need", + "summary": { + "short": "...", + "medium": "...", + "long": "..." + }, + "content": { + "full_text": "[Full PDF text extracted]", + "transcript": null, + "excerpts": [ + "Abstract: [full abstract]", + "\"The Transformer is the first transduction model relying entirely on self-attention\"", + "Key finding: BLEU score of 28.4 on WMT 2014 English-to-German" + ] + }, + "metadata": { + "source_url": "https://arxiv.org/pdf/1706.03762.pdf", + "canonical_url": "https://arxiv.org/abs/1706.03762", + "published_date": "2017-06-12T00:00:00Z", + "accessed_date": "2024-01-16T15:00:00Z", + "language": "en", + "word_count": 7500, + "read_time_minutes": 30, + "author_platform": "arxiv" + } + } +} +``` + +**Set confidence score:** +- Full text extracted, good metadata: 0.8-0.9 +- Partial extraction: 0.5-0.7 +- Metadata only (encrypted PDF): 0.2-0.4 + +--- + +## Error Handling + +**Password-protected PDF:** +``` +⚠️ PDF is password-protected +→ Extract metadata only +→ Confidence: 0.2 +→ Warning: "Cannot extract text - PDF is encrypted" +``` + +**Scanned PDF (images, no text):** +``` +⚠️ PDF contains scanned images +→ Attempt OCR if available +→ If OCR fails, extract metadata only +→ Lower confidence score +→ Warning: "PDF appears to be scanned images, text extraction limited" +``` + +**Corrupted PDF:** +``` +❌ PDF file is corrupted +→ Cannot extract content +→ Log error +→ Return minimal metadata from URL/filename +``` + +--- + +## Output Example + +**Successful extraction (research paper):** +``` +✅ Extracted PDF (research paper) +📄 Title: Attention Is All You Need +✍️ Authors: Vaswani et al. (8 authors) +📅 Published: 2017-06-12 +📝 Content: 7,500 words (15 pages) +👥 People: 8 authors +🏢 Companies: 2 (Google Brain, Google Research) +📚 Citations: 42 references +🎯 Confidence: 0.88 +``` + +**Partial extraction (encrypted PDF):** +``` +⚠️ Extracted PDF metadata only (encrypted) +📄 Title: Proprietary Research Report +📝 Content: Text extraction failed (password-protected) +🎯 Confidence: 0.25 +⚠️ Warning: PDF is encrypted, cannot extract full text +``` + +--- + +**This workflow handles PDF extraction including research papers, reports, and documents.** diff --git a/.opencode/skills/Utilities/Parser/Workflows/ExtractTwitter.md b/.opencode/skills/Utilities/Parser/Workflows/ExtractTwitter.md new file mode 100755 index 00000000..c7be682b --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Workflows/ExtractTwitter.md @@ -0,0 +1,429 @@ +# Twitter/X Thread Extraction Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the ExtractTwitter workflow in the Parser skill to parse tweets"}' \ + > /dev/null 2>&1 & +``` + +Running the **ExtractTwitter** workflow in the **Parser** skill to parse tweets... + +**Purpose:** Extract tweets, threads, and metadata from Twitter/X + +**When to Use:** Content type detected as "tweet_thread" from twitter.com or x.com domains + +--- + +## Extraction Steps + +### 1. Thread Detection and Fetching + +**Determine if single tweet or thread:** +- Fetch initial tweet +- Check for thread indicator (replies from same author) +- Follow thread if present + +**Fetch entire thread:** +- Get all tweets in conversation from same author +- Maintain chronological order +- Include reply context if relevant + +**API alternatives (if Twitter API unavailable):** +- Use Bright Data Twitter scraper (via mcp skill) +- Scrape page HTML directly +- Use third-party Twitter API alternatives +- Try nitter instances as fallback + +--- + +### 2. Tweet Metadata Extraction + +**For each tweet in thread:** +- Tweet ID +- Author username and display name +- Tweet text +- Timestamp +- Engagement metrics (likes, retweets, replies, views) +- Media attachments (images, videos, links) +- Quoted tweets +- @mentions +- #hashtags + +**Thread-level metadata:** +- Total tweets in thread +- Total engagement (sum across all tweets) +- Thread start time +- Thread end time (if completed) + +--- + +### 3. Author Information + +**Extract author details:** +- Username (@handle) +- Display name +- Bio +- Profile URL +- Follower count +- Following count +- Verified status +- Profile picture URL +- Location (if listed) +- Website (if listed) + +**Add to people array:** +```json +{ + "name": "Author Display Name", + "role": "author", + "title": "Title from bio", + "company": "Company from bio", + "social": { + "twitter": "@authorhandle", + "linkedin": "[extracted from bio or pinned tweet]", + "email": null, + "website": "https://authorwebsite.com" + }, + "context": "Twitter thread author discussing [topic]", + "importance": "primary" +} +``` + +--- + +### 4. Thread Reconstruction + +**Combine thread into coherent content:** + +**Concatenate tweets:** +- Join all tweets in chronological order +- Preserve paragraph breaks +- Mark tweet boundaries if needed + +**Example:** +``` +Tweet 1/10: Let me explain why GPT-4 is such a big deal 🧵 + +Tweet 2/10: First, the multimodal capabilities mean it can process both text and images natively. This opens up entirely new use cases. + +Tweet 3/10: Second, the reasoning improvements are substantial. On the bar exam, it scores in the 90th percentile vs GPT-3.5's 10th percentile. + +[...continue for all tweets...] +``` + +**Full text with markers:** +``` +[1/10] Let me explain why GPT-4 is such a big deal 🧵 + +[2/10] First, the multimodal capabilities... + +[3/10] Second, the reasoning improvements... +``` + +**Clean text (no markers):** +``` +Let me explain why GPT-4 is such a big deal. First, the multimodal capabilities mean it can process both text and images natively. This opens up entirely new use cases. Second, the reasoning improvements are substantial... +``` + +--- + +### 5. Link Extraction + +**Extract all URLs from thread:** + +**Types of links:** +- External links (articles, websites) +- Twitter media (images, videos) +- Quoted tweets +- Reply context tweets +- Shortened URLs (expand t.co links) + +**For each link:** +```json +{ + "url": "https://openai.com/blog/gpt-4", + "domain": "openai.com", + "title": "GPT-4 announcement", + "description": "Linked in tweet 4/10 when discussing capabilities", + "link_type": "reference", + "context": "Primary source for claims about GPT-4 performance", + "position": "middle" +} +``` + +--- + +### 6. Mention and Hashtag Extraction + +**Extract @mentions:** +- All mentioned users +- Context of mention (quoting, referencing, replying) +- Add mentioned people to people array if relevant + +**Example:** +```json +{ + "name": "Sam Altman", + "role": "mentioned", + "title": "CEO of OpenAI", + "company": "OpenAI", + "social": { + "twitter": "@sama", + "linkedin": null, + "email": null, + "website": null + }, + "context": "Mentioned when discussing GPT-4 announcement", + "importance": "secondary" +} +``` + +**Extract #hashtags:** +- Add to tags array +- Use for topic classification + +--- + +### 7. Media Extraction + +**Extract media attachments:** +- Images (URLs to Twitter media) +- Videos (URLs, may need special handling) +- GIFs + +**Describe media in context:** +- Include image descriptions in excerpts +- Note if media is core to thread (e.g., charts, screenshots) + +--- + +### 8. Gemini Analysis + +**Send thread content to Gemini:** + +**Extract:** +- Main topic/thesis of thread +- Key arguments and points +- People and companies mentioned (beyond @mentions) +- Technical concepts discussed +- Sentiment and tone + +**Generate summaries:** + +**Short (1-2 sentences):** +``` +Thread explaining GPT-4's significance, covering multimodal capabilities, reasoning improvements, and implications for AI development. +``` + +**Medium (paragraph):** +``` +Comprehensive thread breaking down why GPT-4 represents a major advancement in AI. Covers three main areas: multimodal capabilities enabling text+image processing, substantial reasoning improvements (90th percentile on bar exam), and architectural innovations. Author argues this shifts the landscape for AI applications and discusses potential use cases in education, healthcare, and software development. Includes benchmarks, examples, and links to official announcements. +``` + +**Long:** +``` +[Detailed summary with all key points, arguments, evidence, and conclusions] +``` + +**Key excerpts:** +- Notable quotes from thread +- Important statistics or claims +- Surprising insights + +--- + +### 9. Topic Classification + +**Analyze thread for categorization:** +- Primary topic (AI, tech, politics, etc.) +- Secondary topics +- Extract hashtags as tags +- Identify keywords from content +- Map to newsletter sections + +**Example:** +```json +{ + "primary_category": "AI", + "secondary_categories": ["technology", "business"], + "tags": [ + "gpt-4", + "openai", + "multimodal-ai", + "ai-benchmarks", + "language-models" + ], + "keywords": [ + "GPT-4", + "multimodal", + "reasoning", + "bar exam", + "benchmarks" + ], + "themes": [ + "AI capability improvements", + "Multimodal AI emergence", + "Real-world AI applications" + ], + "newsletter_sections": ["AI", "Headlines"] +} +``` + +--- + +### 10. Engagement Analysis + +**Analyze engagement metrics:** +- Total likes across thread +- Total retweets +- Total replies +- Views (if available) +- Engagement rate + +**Use for trending potential:** +- High engagement (10k+ likes): High trending potential +- Medium engagement (1k-10k): Medium +- Low engagement (<1k): Low + +**Factor into importance score:** +- Viral threads (100k+ engagement): 9-10 +- Popular threads (10k-100k): 7-9 +- Moderate threads (1k-10k): 5-7 +- Small threads (<1k): 3-5 + +--- + +### 11. Schema Population + +**Populate content section:** +```json +{ + "content": { + "id": "uuid-generated", + "type": "tweet_thread", + "title": "Why GPT-4 is a Big Deal (10-tweet thread)", + "summary": { + "short": "Thread explaining GPT-4's significance and implications", + "medium": "[paragraph summary]", + "long": "[comprehensive summary]" + }, + "content": { + "full_text": "[Full thread text concatenated]", + "transcript": null, + "excerpts": [ + "\"GPT-4 scores in the 90th percentile on the bar exam\"", + "Key point about multimodal capabilities", + "Surprising insight about use cases" + ] + }, + "metadata": { + "source_url": "https://twitter.com/user/status/123456", + "canonical_url": "https://twitter.com/user/status/123456", + "published_date": "2024-01-15T14:30:00Z", + "accessed_date": "2024-01-16T16:00:00Z", + "language": "en", + "word_count": 850, + "read_time_minutes": 3, + "author_platform": "twitter" + } + }, + "analysis": { + "sentiment": "positive", + "importance_score": 8, + "novelty_score": 7, + "controversy_score": 4, + "relevance_to_audience": ["ai_researchers", "technologists", "general_tech"], + "key_insights": [ + "GPT-4 represents step-change in reasoning ability", + "Multimodal capabilities enable new applications", + "Performance improvements validated on standardized tests" + ], + "related_content_ids": [], + "trending_potential": "high" + } +} +``` + +**Engagement metadata (custom field for tweets):** +```json +// Could add to extraction_metadata or custom field +"twitter_metrics": { + "likes": 15000, + "retweets": 3000, + "replies": 500, + "views": 250000, + "thread_length": 10 +} +``` + +--- + +## Error Handling + +**Private/protected account:** +``` +❌ Tweet is from private account +→ Cannot access content +→ Return minimal metadata (author handle, tweet ID) +→ Confidence: 0.1 +→ Warning: "Tweet is from protected account, content unavailable" +``` + +**Deleted tweet:** +``` +❌ Tweet has been deleted +→ Check archive.org or cached versions +→ If unavailable, return error +→ Warning: "Tweet no longer available" +``` + +**Rate limiting:** +``` +⚠️ Twitter API rate limit reached +→ Back off and retry +→ Use alternative scraping methods +→ May reduce data quality +→ Warning: "Rate limited, using fallback extraction" +``` + +**Thread not fully loaded:** +``` +⚠️ Could not fetch entire thread +→ Extract partial thread +→ Note: "Partial thread (X of Y tweets)" +→ Lower confidence score +``` + +--- + +## Output Example + +**Successful extraction:** +``` +✅ Extracted Twitter thread +🐦 Author: @username (Display Name) +📝 Thread: 10 tweets, 850 words +💬 Engagement: 15k likes, 3k retweets, 500 replies +🔗 Links: 5 +👥 Mentions: 3 +#️⃣ Hashtags: 4 +🎯 Confidence: 0.85 +🔥 Trending potential: High +``` + +**Partial extraction:** +``` +⚠️ Extracted Twitter thread (partial) +🐦 Author: @username +📝 Thread: 6 of 10 tweets (some tweets unavailable) +🔗 Links: 3 +🎯 Confidence: 0.60 +⚠️ Warning: Could not fetch complete thread +``` + +--- + +**This workflow handles Twitter/X thread extraction with engagement metrics and thread reconstruction.** diff --git a/.opencode/skills/Utilities/Parser/Workflows/ExtractYoutube.md b/.opencode/skills/Utilities/Parser/Workflows/ExtractYoutube.md new file mode 100755 index 00000000..926d0cbd --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Workflows/ExtractYoutube.md @@ -0,0 +1,386 @@ +# YouTube Video Extraction Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the ExtractYoutube workflow in the Parser skill to parse videos"}' \ + > /dev/null 2>&1 & +``` + +Running the **ExtractYoutube** workflow in the **Parser** skill to parse videos... + +**Purpose:** Extract transcript, metadata, and entities from YouTube videos + +**When to Use:** Content type detected as "video" from YouTube domains + +--- + +## Extraction Steps + +### 1. Transcript Extraction (Fabric) + +**Use Fabric yt pattern:** +```bash +yt --transcript "https://youtube.com/watch?v=VIDEO_ID" +``` + +**Output:** +- Full video transcript with timestamps +- Cleaned text (no overlapping timestamps) +- Speaker labels if available + +**If transcript unavailable:** +- Check if auto-captions are disabled +- Check if video is private/restricted +- Log warning: "No transcript available" +- Proceed with metadata-only extraction +- Set confidence_score to 0.4 or lower + +--- + +### 2. Page Metadata Scraping + +**Scrape YouTube page for metadata:** + +**Required fields:** +- Title: `` +- Description: `` +- Publish date: `` +- Channel name: `` or from page +- Channel URL: Extract from page +- Video duration: `` +- View count: Extract from page (if available) +- Thumbnail: `` + +**Optional fields:** +- Tags/keywords: `` +- Category: Extract from page +- License: Standard YouTube License or Creative Commons + +**Example metadata extraction:** +```typescript +{ + title: "Building AGI: Interview with Anthropic CEO", + description: "Deep dive into AI safety and scaling...", + published_date: "2024-01-15T10:30:00Z", + channel_name: "AI Podcast", + channel_url: "https://youtube.com/@aipodcast", + duration: "PT1H23M45S", // ISO 8601 duration + view_count: 125000, + thumbnail: "https://i.ytimg.com/vi/VIDEO_ID/maxresdefault.jpg" +} +``` + +--- + +### 3. Channel Information Extraction + +**Extract channel owner details:** + +**From channel page (if accessible):** +- Channel owner name +- Channel description +- Social links (Twitter, website, etc.) +- Subscriber count +- Verified status + +**Add to people array:** +```json +{ + "name": "Channel Owner Name", + "role": "author", + "title": "YouTuber / Podcast Host", + "company": null, + "social": { + "twitter": "@channelowner", + "linkedin": null, + "email": null, + "website": "https://channelwebsite.com" + }, + "context": "YouTube channel owner and video creator", + "importance": "primary" +} +``` + +--- + +### 4. Description Link Parsing + +**Extract all links from video description:** + +**Common link types in descriptions:** +- Timestamps (e.g., "0:00 Introduction") +- Referenced websites +- Sponsor links +- Social media links +- Related videos/playlists +- Affiliate links + +**For each link:** +```json +{ + "url": "https://example.com/resource", + "domain": "example.com", + "title": "Resource mentioned in video", + "description": "Tool/article/paper discussed", + "link_type": "reference", + "context": "Mentioned at 12:34 in discussion of X", + "position": "beginning" // based on timestamp +} +``` + +--- + +### 5. Gemini Analysis + +**Send transcript to Gemini for entity extraction:** + +**Prompt:** Use `prompts/entity-extraction.md` + +**Extract from transcript:** +- All people mentioned (speakers, subjects, guests) +- Companies/organizations discussed +- Products/tools mentioned +- Technical concepts and topics + +**People extraction from transcript:** +```json +{ + "name": "Dario Amodei", + "role": "subject", + "title": "CEO of Anthropic", + "company": "Anthropic", + "social": { + "twitter": "@darioamodei", + "linkedin": "...", + "email": null, + "website": null + }, + "context": "Interviewed about AI safety and scaling laws", + "importance": "primary" +} +``` + +**Company extraction:** +```json +{ + "name": "Anthropic", + "domain": "anthropic.com", + "industry": "AI", + "context": "Subject of interview, discussed AI safety approach", + "mentioned_as": "subject", + "sentiment": "positive" +} +``` + +--- + +### 6. Summarization + +**Use Gemini to generate summaries:** + +**Prompt:** Use `prompts/summarization.md` + +**Generate three levels:** + +**Short (1-2 sentences):** +``` +Interview with Anthropic CEO discussing AI safety, scaling laws, and the path to beneficial AGI. +``` + +**Medium (paragraph):** +``` +Deep dive conversation with Dario Amodei, CEO of Anthropic, covering the company's approach to AI safety through constitutional AI, recent progress with Claude models, thoughts on scaling laws and their limits, and the challenges of building AGI that remains beneficial and aligned with human values. Discussion includes technical details on RLHF, interpretability research, and predictions for the next 5 years of AI development. +``` + +**Long (multiple paragraphs):** +``` +[Comprehensive summary with key points, main arguments, notable quotes, and takeaways] +``` + +**Extract key excerpts:** +- Notable quotes from speakers +- Important technical explanations +- Surprising insights or predictions + +--- + +### 7. Topic Classification + +**Analyze transcript for topics:** + +**Primary category:** +- Determine main subject (AI, security, technology, etc.) + +**Secondary categories:** +- Related topics discussed + +**Tags:** +- Specific technical terms (e.g., "RLHF", "scaling laws", "constitutional AI") +- People mentioned +- Companies mentioned + +**Newsletter sections:** +- Map to UL sections (e.g., "AI", "Headlines", "Analysis") + +**Example:** +```json +{ + "primary_category": "AI", + "secondary_categories": ["technology", "business", "research"], + "tags": [ + "artificial-intelligence", + "ai-safety", + "anthropic", + "claude", + "rlhf", + "constitutional-ai", + "scaling-laws", + "agi" + ], + "keywords": [ + "AI safety", + "constitutional AI", + "RLHF", + "scaling laws", + "alignment", + "interpretability" + ], + "themes": [ + "Building safe AGI", + "Scaling vs safety tradeoffs", + "Future of AI development" + ], + "newsletter_sections": ["AI", "Analysis", "Headlines"] +} +``` + +--- + +### 8. Content Analysis + +**Analyze video for:** + +**Sentiment:** +- Overall tone (positive about AI safety, cautious about risks, etc.) + +**Importance score (1-10):** +- Based on: Channel authority, view count, topic relevance +- High (8-10): Major announcement, industry leader interview +- Medium (5-7): Informative content, mid-tier channel +- Low (1-4): Opinion piece, small channel + +**Novelty score (1-10):** +- How new/unique is the information? +- Original research/announcement: 8-10 +- New perspective on known topic: 5-7 +- Recap of existing knowledge: 1-4 + +**Controversy score (1-10):** +- How polarizing or debatable? +- Highly controversial takes: 8-10 +- Some debate: 5-7 +- Widely accepted: 1-4 + +**Trending potential:** +- High: Viral topic, major news, controversial +- Medium: Interesting but niche +- Low: Evergreen content, small audience + +--- + +### 9. Schema Population + +**Populate content section:** +```json +{ + "content": { + "id": "uuid-generated", + "type": "video", + "title": "Building AGI: Interview with Anthropic CEO", + "summary": { + "short": "...", + "medium": "...", + "long": "..." + }, + "content": { + "full_text": null, + "transcript": "[Full video transcript]", + "excerpts": ["Notable quote 1", "Notable quote 2"] + }, + "metadata": { + "source_url": "https://youtube.com/watch?v=abc123", + "canonical_url": "https://youtube.com/watch?v=abc123", + "published_date": "2024-01-15T10:30:00Z", + "accessed_date": "2024-01-16T14:22:00Z", + "language": "en", + "word_count": 8500, // transcript word count + "read_time_minutes": 83, // video duration in minutes + "author_platform": "youtube" + } + } +} +``` + +**Set confidence score:** +- Transcript available + good metadata: 0.85-0.95 +- No transcript but good metadata: 0.4-0.6 +- Limited metadata: 0.3-0.5 + +--- + +## Error Handling + +**No transcript available:** +- Log warning: "Transcript unavailable - captions disabled" +- Extract metadata only +- Use video description for some context +- Lower confidence score to 0.4 + +**Private/restricted video:** +- Log error: "Video is private or restricted" +- Extract minimal public metadata +- Confidence score: 0.2 + +**Geo-blocked content:** +- Attempt to fetch via VPN/proxy if available +- Otherwise log warning +- Extract public metadata only + +**Age-restricted content:** +- May require authenticated request +- Fall back to public metadata if auth unavailable + +--- + +## Output Example + +**Successful extraction:** +``` +✅ Extracted YouTube video +📺 Title: Building AGI: Interview with Anthropic CEO +⏱️ Duration: 1h 23m +📝 Transcript: 8,500 words +👥 People: 2 (Dario Amodei, Interviewer) +🏢 Companies: 3 (Anthropic, OpenAI, Google) +🔗 Links: 7 (from description) +🎯 Confidence: 0.92 +``` + +**Partial extraction (no transcript):** +``` +⚠️ Extracted YouTube video (metadata only - no transcript) +📺 Title: Building AGI: Interview with Anthropic CEO +⏱️ Duration: 1h 23m +📝 Transcript: Not available +👥 People: 1 (channel owner) +🔗 Links: 5 (from description) +🎯 Confidence: 0.45 +⚠️ Warning: Captions disabled, transcript unavailable +``` + +--- + +**This workflow handles comprehensive YouTube video extraction with Fabric and Gemini.** diff --git a/.opencode/skills/Utilities/Parser/Workflows/ParseContent.md b/.opencode/skills/Utilities/Parser/Workflows/ParseContent.md new file mode 100755 index 00000000..e241ed30 --- /dev/null +++ b/.opencode/skills/Utilities/Parser/Workflows/ParseContent.md @@ -0,0 +1,413 @@ +# Parse Content Workflow + +## Voice Notification + +```bash +curl -s -X POST http://localhost:8888/notify \ + -H "Content-Type: application/json" \ + -d '{"message": "Running the ParseContent workflow in the Parser skill to parse URLs"}' \ + > /dev/null 2>&1 & +``` + +Running the **ParseContent** workflow in the **Parser** skill to parse URLs... + +**Purpose:** Main orchestration workflow for parsing any URL into deterministic JSON schema + +**When to Use:** User provides one or more URLs for parsing into structured data + +--- + +## Workflow Steps + +### 1. Input Validation + +**Receive URL(s) from User:** +- Single URL: `"Parse this: https://example.com/article"` +- Multiple URLs: List of URLs (newline or comma-separated) +- Batch file: Path to text file with URLs (one per line) + +**Validate inputs:** +- Check URL format (must be valid HTTP/HTTPS) +- Test URL accessibility (returns 200 OK) +- Detect redirects and follow to canonical URL +- Log any inaccessible URLs for error reporting + +--- + +### 2. Content Type Detection + +**Route to:** `Workflows/Detect-content-type.md` + +**For each URL, determine:** +- YouTube video (youtube.com, youtu.be domains) +- PDF document (*.pdf extension or Content-Type: application/pdf) +- Twitter/X thread (twitter.com, x.com domains with /status/ path) +- Newsletter (substack.com, beehiiv.com, convertkit domains) +- Web article (generic HTML content) + +**Detection priority:** +1. Domain-based detection (YouTube, Twitter, known platforms) +2. Extension-based detection (PDF, HTML) +3. Content-Type header inspection +4. Default to "generic" web article if uncertain + +--- + +### 3. Specialized Extraction + +**Route to appropriate extractor based on content type:** + +**YouTube Videos** → `Workflows/extract/Youtube.md` +- Use Fabric `yt --transcript` for transcript +- Scrape page for metadata +- Extract channel information +- Parse description for links + +**Web Articles** → `Workflows/extract/Article.md` +- Use Gemini Researcher for full content scraping +- Extract HTML metadata (OpenGraph, Twitter Cards, JSON-LD) +- Parse article body for entities and links +- Identify author, publication, date + +**PDF Documents** → `Workflows/extract/Pdf.md` +- Extract text content from PDF +- Parse metadata (author, title, date) +- Use Gemini for entity extraction +- Handle citations and references + +**Newsletter HTML** → `Workflows/extract/Newsletter.md` +- Parse HTML structure +- Extract individual sections/stories +- Identify links with context +- Preserve formatting for excerpts + +**Twitter/X Threads** → `Workflows/extract/Twitter.md` +- Fetch entire thread +- Combine tweets into coherent content +- Extract @mentions and #hashtags +- Parse engagement metrics + +**Generic Web Pages** → `Workflows/extract/Article.md` (fallback) +- Extract whatever content is available +- Lower confidence scores +- Flag as "generic" type + +--- + +### 4. Gemini Analysis + +**For ALL content types, use Gemini Researcher to extract:** + +**People Extraction** (use `prompts/entity-extraction.md`) +- Identify all people mentioned +- Determine roles (author, subject, quoted, etc.) +- Find social handles and affiliations +- Assess importance (primary/secondary/minor) + +**Company Extraction** (use `prompts/entity-extraction.md`) +- Identify all companies/organizations +- Extract domains and industry classifications +- Determine mention context (subject, competitor, partner, etc.) +- Assess sentiment (positive, neutral, negative, mixed) + +**Topic Classification** (use `prompts/topic-classification.md`) +- Determine primary and secondary categories +- Extract tags, keywords, themes +- Map to newsletter sections (Headlines, Analysis, Tools, etc.) +- Identify audience relevance + +**Summarization** (use `prompts/summarization.md`) +- Generate short summary (1-2 sentences, Twitter-length) +- Generate medium summary (paragraph, newsletter preview) +- Generate long summary (comprehensive, multiple paragraphs) +- Extract key excerpts/quotes + +**Link Analysis** (use `prompts/link-analysis.md`) +- Identify all URLs in content +- Classify link types (reference, source, related, tool, etc.) +- Extract context for each link +- Determine position in content + +**Content Analysis:** +- Overall sentiment (positive, neutral, negative, mixed) +- Importance score (1-10 scale) +- Novelty score (1-10, how new/unique) +- Controversy score (1-10, how polarizing) +- Trending potential (low, medium, high) +- Key insights extraction + +--- + +### 5. Schema Population + +**Generate UUID:** +- Create UUID v4 for content.id +- Ensures uniqueness across all parsed content + +**Populate all 9 schema sections:** + +1. **content**: Core content data + - id, type, title + - summary (short, medium, long) + - content (full_text, transcript, excerpts) + - metadata (URLs, dates, language, word count, platform) + +2. **people**: Array of person objects + - name, role, title, company + - social handles (twitter, linkedin, email, website) + - context and importance + +3. **companies**: Array of company objects + - name, domain, industry + - context, mentioned_as, sentiment + +4. **topics**: Topic classification + - primary_category, secondary_categories + - tags, keywords, themes + - newsletter_sections + +5. **links**: Array of link objects + - url, domain, title, description + - link_type, context, position + +6. **sources**: Array of source objects + - publication, author, url + - published_date, source_type + +7. **newsletter_metadata**: UL-specific fields + - issue_number (null for new content) + - section (null, User assigns later) + - position_in_section (null) + - editorial_note (null, User adds later) + - include_in_newsletter (false by default) + - scheduled_date (null) + +8. **analysis**: Automated analysis + - sentiment + - importance_score, novelty_score, controversy_score + - relevance_to_audience (array) + - key_insights (array) + - related_content_ids (empty array initially) + - trending_potential + +9. **extraction_metadata**: Processing info + - processed_date (current timestamp) + - processing_method (gemini, fabric, hybrid) + - confidence_score (0-1 scale) + - warnings (array of any issues) + - version ("1.0.0") + +**Schema Invariants:** +- ALL fields must be present (use null for missing data) +- NEVER omit schema fields +- Arrays can be empty but must exist +- Dates in ISO 8601 format +- Confidence score reflects extraction quality + +--- + +### 6. Validation + +**Validate against schema:** +- Load `schema/content-schema.json` +- Validate populated data against schema +- Check all required fields present +- Verify data types match schema +- Ensure enums use allowed values + +**If validation fails:** +- Log validation errors to warnings array +- Attempt to fix common issues (type coercion, etc.) +- Re-validate after fixes +- If still fails, output with low confidence score + +**Quality checks:** +- Title non-empty +- At least short summary populated +- Source URL present +- Processed date valid +- UUID format correct + +--- + +### 7. Output Generation + +**Generate filename:** +- Format: `{timestamp}_{sanitized-title}.json` +- timestamp: `YYYYMMDD-HHMMSS` format +- sanitized-title: lowercase, replace spaces with hyphens, remove special chars +- Example: `20251114-153045_ai-breakthrough-anthropic.json` + +**Write JSON file:** +- Pretty-print with 2-space indentation +- Location: Current directory (or User-specified path) +- UTF-8 encoding +- Ensure valid JSON syntax + +**Output to User:** +``` +✅ Parsed: [title] +📄 Output: [filename] +📊 Stats: [word_count] words, [people_count] people, [company_count] companies, [link_count] links +🎯 Confidence: [confidence_score] +⚠️ Warnings: [warnings if any] +``` + +--- + +### 8. Batch Processing + +**For multiple URLs:** + +**Sequential processing:** +1. Process URLs one at a time +2. Update User after each completion +3. Continue even if one fails +4. Collect all results and errors + +**Progress updates:** +``` +Processing 1 of 5: [URL] +✅ Completed: [filename] + +Processing 2 of 5: [URL] +❌ Failed: [error message] + +... +``` + +**Summary report:** +``` +📊 Batch Processing Complete +✅ Successful: 4/5 +❌ Failed: 1/5 + +Output files: +- 20251114-153045_article-1.json +- 20251114-153120_video-transcript.json +- 20251114-153215_research-paper.json +- 20251114-153305_twitter-thread.json + +Failed: +- https://example.com/broken (404 Not Found) +``` + +--- + +## Error Handling + +**Network errors:** +- Retry up to 3 times with exponential backoff +- Log error details to warnings +- Set confidence_score to 0.3 or lower +- Output partial data if any was extracted + +**Parsing errors:** +- Fall back to generic extraction +- Populate what's possible +- Flag in warnings array +- Never fail completely - always output valid JSON + +**Gemini API errors:** +- Retry with exponential backoff +- Fall back to simpler extraction methods +- Reduce confidence score accordingly +- Log API error in warnings + +**Unsupported content:** +- Default to generic web page extraction +- Flag as "generic" content type +- Extract basic metadata (title, URL, date) +- Set confidence_score to 0.5 or lower + +--- + +## Quality Assurance + +**Before outputting JSON:** + +1. **Completeness check:** + - All 9 schema sections present + - All required fields populated (or null) + - No undefined values + +2. **Data quality check:** + - Title makes sense + - Summaries coherent + - People/companies have names + - Links have valid URLs + - Dates in ISO 8601 format + +3. **Confidence assessment:** + - High (0.8-1.0): Complete extraction, verified data + - Medium (0.5-0.79): Most data extracted, some gaps + - Low (0.0-0.49): Partial extraction, many uncertainties + +4. **Warning review:** + - Log any extraction issues + - Note missing fields + - Flag low-confidence extractions + - Document any assumptions made + +--- + +## Usage Examples + +**Example 1: Single Article** +``` +User: "Parse this article: https://anthropic.com/news/claude-3" + +{DAIDENTITY.NAME}: +1. Detects web article type +2. Uses Gemini to scrape and analyze +3. Extracts entities, topics, links +4. Generates summaries +5. Populates schema +6. Validates and outputs JSON + +✅ Parsed: Claude 3 Model Family Announcement +📄 Output: 20251114-153045_claude-3-announcement.json +📊 Stats: 2,500 words, 5 people, 3 companies, 12 links +🎯 Confidence: 0.92 +``` + +**Example 2: YouTube Video** +``` +User: "Parse this YouTube video: https://youtube.com/watch?v=abc123" + +{DAIDENTITY.NAME}: +1. Detects YouTube video type +2. Uses Fabric for transcript extraction +3. Scrapes video metadata +4. Gemini analyzes transcript for entities +5. Extracts channel info and description links +6. Outputs JSON + +✅ Parsed: AI Safety Interview with Anthropic CEO +📄 Output: 20251114-153120_ai-safety-interview.json +📊 Stats: 8,000 words (transcript), 3 people, 5 companies, 7 links +🎯 Confidence: 0.88 +``` + +**Example 3: Batch Processing** +``` +User: "Parse these 3 URLs: +- https://arxiv.org/pdf/2401.12345 +- https://twitter.com/user/status/123 +- https://newsletter.com/issue/42" + +{DAIDENTITY.NAME}: +[Processes each sequentially] + +📊 Batch Processing Complete +✅ Successful: 3/3 + +Output files: +- 20251114-153215_attention-all-you-need.json (PDF) +- 20251114-153340_ai-breakthrough-thread.json (Twitter) +- 20251114-153425_weekly-ai-news.json (Newsletter) +``` + +--- + +**This workflow handles all content parsing requests with deterministic JSON output.** diff --git a/.opencode/skills/Utilities/Parser/entity-index.json b/.opencode/skills/Utilities/Parser/entity-index.json new file mode 100755 index 00000000..d3ed53e3 --- /dev/null +++ b/.opencode/skills/Utilities/Parser/entity-index.json @@ -0,0 +1,8 @@ +{ + "version": "1.0.0", + "last_updated": "2025-11-14T18:00:00Z", + "people": {}, + "companies": {}, + "links": {}, + "sources": {} +} diff --git a/.opencode/skills/Utilities/Pdf/LICENSE.txt b/.opencode/skills/Utilities/Pdf/LICENSE.txt new file mode 100755 index 00000000..c55ab422 --- /dev/null +++ b/.opencode/skills/Utilities/Pdf/LICENSE.txt @@ -0,0 +1,30 @@ +© 2025 Anthropic, PBC. All rights reserved. + +LICENSE: Use of these materials (including all code, prompts, assets, files, +and other components of this Skill) is governed by your agreement with +Anthropic regarding use of Anthropic's services. If no separate agreement +exists, use is governed by Anthropic's Consumer Terms of Service or +Commercial Terms of Service, as applicable: +https://www.anthropic.com/legal/consumer-terms +https://www.anthropic.com/legal/commercial-terms +Your applicable agreement is referred to as the "Agreement." "Services" are +as defined in the Agreement. + +ADDITIONAL RESTRICTIONS: Notwithstanding anything in the Agreement to the +contrary, users may not: + +- Extract these materials from the Services or retain copies of these + materials outside the Services +- Reproduce or copy these materials, except for temporary copies created + automatically during authorized use of the Services +- Create derivative works based on these materials +- Distribute, sublicense, or transfer these materials to any third party +- Make, offer to sell, sell, or import any inventions embodied in these + materials +- Reverse engineer, decompile, or disassemble these materials + +The receipt, viewing, or possession of these materials does not convey or +imply any license or right beyond those expressly granted above. + +Anthropic retains all right, title, and interest in these materials, +including all copyrights, patents, and other intellectual property rights. diff --git a/.opencode/skills/Utilities/Pdf/SKILL.md b/.opencode/skills/Utilities/Pdf/SKILL.md new file mode 100755 index 00000000..6ed4ca6a --- /dev/null +++ b/.opencode/skills/Utilities/Pdf/SKILL.md @@ -0,0 +1,440 @@ +--- +name: Pdf +description: PDF processing. USE WHEN pdf, PDF file. SkillSearch('pdf') for docs. +--- + +# PDF Processing Guide + +## 🎯 Load Full PAI Context + +**Before starting any task with this skill, load complete PAI context:** + +`read ~/.opencode/skills/PAI/SKILL.md` + +This provides access to: +- Complete contact list (Angela, Bunny, Saša, Greg, team members) +- Stack preferences (TypeScript>Python, bun>npm, uv>pip) +- Security rules and repository safety protocols +- Response format requirements (structured emoji format) +- Voice IDs for agent routing (ElevenLabs) +- Personal preferences and operating instructions + +## When to Activate This Skill + +### Direct PDF Task Triggers +- User wants to **create** a new PDF document +- User wants to **merge**, **combine**, or **concatenate** multiple PDFs +- User wants to **split** or **separate** a PDF into individual pages/sections +- User mentions "**extract text from PDF**", "**PDF text extraction**" +- User mentions "**extract tables from PDF**", "**PDF tables**" +- User wants to "**fill PDF form**", "**PDF form filling**" +- User mentions "**OCR**", "**scanned PDF**", or "**scan to text**" +- User wants to add **watermarks**, **password protection**, or **encryption** +- User wants to **extract images** from a PDF +- User wants to **rotate pages** or manipulate PDF structure + +### Contextual Triggers +- User provides a **.pdf file path** for processing +- User mentions form filling automation or batch PDF processing +- User needs to process PDFs programmatically at scale + +## 🔀 PDF Workflow Routing + +This skill supports multiple PDF processing workflows: + +### Creation Workflow +**Trigger:** "create PDF", "generate PDF", "make PDF", "PDF from data" + +**Tools:** reportlab (Python) +**Documentation:** Lines 136-181 (SKILL.md) + +**Use Cases:** +- Creating new PDFs from scratch +- Generating reports programmatically +- Multi-page documents with text and graphics +- PDF generation from templates or data + +### Merge/Split Workflow +**Trigger:** "merge PDFs", "combine PDFs", "split PDF", "separate pages" + +**Tools:** pypdf (Python), qpdf (CLI) +**Documentation:** Lines 46-68 (SKILL.md), Lines 199-211 (qpdf) + +**Use Cases:** +- Combining multiple PDFs into one document +- Splitting PDFs into individual pages or ranges +- Reorganizing PDF page order +- Extracting specific page ranges + +### Text Extraction Workflow +**Trigger:** "extract text", "PDF to text", "read PDF content" + +**Tools:** pdfplumber (Python), pdftotext (CLI) +**Documentation:** Lines 95-103 (pdfplumber), Lines 186-196 (pdftotext) + +**Use Cases:** +- Extracting text while preserving layout +- Converting PDFs to plain text +- Batch text extraction from multiple PDFs +- Metadata extraction + +### Table Extraction Workflow +**Trigger:** "extract tables", "PDF tables", "table data from PDF" + +**Tools:** pdfplumber + pandas (Python) +**Documentation:** Lines 106-133 (SKILL.md) + +**Use Cases:** +- Extracting structured table data to Excel/CSV +- Financial data extraction from PDF reports +- Converting PDF tables to dataframes +- Multi-table extraction and combination + +### Form Filling Workflow +**Trigger:** "fill PDF form", "PDF form filling", "complete PDF form" + +**Tools:** pdf-lib (JavaScript) or pypdf (Python) +**Documentation:** forms.md (complete guide) + +**Use Cases:** +- Programmatic form completion +- Batch form processing +- Template-based PDF generation +- Form field population from data sources + +### OCR Workflow +**Trigger:** "OCR", "scanned PDF", "extract text from scan", "image to text" + +**Tools:** pytesseract + pdf2image (Python) +**Documentation:** Lines 227-244 (SKILL.md) + +**Use Cases:** +- Extracting text from scanned documents +- Processing image-based PDFs +- Converting scanned forms to editable text +- Legacy document digitization + +### Manipulation Workflow +**Trigger:** "watermark", "password protect", "encrypt PDF", "rotate pages", "extract images" + +**Tools:** pypdf (Python), pdfimages (CLI) +**Documentation:** Lines 246-288 (SKILL.md) + +**Use Cases:** +- Adding watermarks to PDFs +- Password protection and encryption +- Page rotation and transformation +- Image extraction from PDFs + +## Overview + +This guide covers essential PDF processing operations using Python libraries and command-line tools. For advanced features, JavaScript libraries, and detailed examples, see reference.md. If you need to fill out a PDF form, read forms.md and follow its instructions. + +## Quick Start + +```python +from pypdf import PdfReader, PdfWriter + +# Read a PDF +reader = PdfReader("document.pdf") +print(f"Pages: {len(reader.pages)}") + +# Extract text +text = "" +for page in reader.pages: + text += page.extract_text() +``` + +## Python Libraries + +### pypdf - Basic Operations + +#### Merge PDFs +```python +from pypdf import PdfWriter, PdfReader + +writer = PdfWriter() +for pdf_file in ["doc1.pdf", "doc2.pdf", "doc3.pdf"]: + reader = PdfReader(pdf_file) + for page in reader.pages: + writer.add_page(page) + +with open("merged.pdf", "wb") as output: + writer.write(output) +``` + +#### Split PDF +```python +reader = PdfReader("input.pdf") +for i, page in enumerate(reader.pages): + writer = PdfWriter() + writer.add_page(page) + with open(f"page_{i+1}.pdf", "wb") as output: + writer.write(output) +``` + +#### Extract Metadata +```python +reader = PdfReader("document.pdf") +meta = reader.metadata +print(f"Title: {meta.title}") +print(f"Author: {meta.author}") +print(f"Subject: {meta.subject}") +print(f"Creator: {meta.creator}") +``` + +#### Rotate Pages +```python +reader = PdfReader("input.pdf") +writer = PdfWriter() + +page = reader.pages[0] +page.rotate(90) # Rotate 90 degrees clockwise +writer.add_page(page) + +with open("rotated.pdf", "wb") as output: + writer.write(output) +``` + +### pdfplumber - Text and Table Extraction + +#### Extract Text with Layout +```python +import pdfplumber + +with pdfplumber.open("document.pdf") as pdf: + for page in pdf.pages: + text = page.extract_text() + print(text) +``` + +#### Extract Tables +```python +with pdfplumber.open("document.pdf") as pdf: + for i, page in enumerate(pdf.pages): + tables = page.extract_tables() + for j, table in enumerate(tables): + print(f"Table {j+1} on page {i+1}:") + for row in table: + print(row) +``` + +#### Advanced Table Extraction +```python +import pandas as pd + +with pdfplumber.open("document.pdf") as pdf: + all_tables = [] + for page in pdf.pages: + tables = page.extract_tables() + for table in tables: + if table: # Check if table is not empty + df = pd.DataFrame(table[1:], columns=table[0]) + all_tables.append(df) + +# Combine all tables +if all_tables: + combined_df = pd.concat(all_tables, ignore_index=True) + combined_df.to_excel("extracted_tables.xlsx", index=False) +``` + +### reportlab - Create PDFs + +#### Basic PDF Creation +```python +from reportlab.lib.pagesizes import letter +from reportlab.pdfgen import canvas + +c = canvas.Canvas("hello.pdf", pagesize=letter) +width, height = letter + +# Add text +c.drawString(100, height - 100, "Hello World!") +c.drawString(100, height - 120, "This is a PDF created with reportlab") + +# Add a line +c.line(100, height - 140, 400, height - 140) + +# Save +c.save() +``` + +#### Create PDF with Multiple Pages +```python +from reportlab.lib.pagesizes import letter +from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, PageBreak +from reportlab.lib.styles import getSampleStyleSheet + +doc = SimpleDocTemplate("report.pdf", pagesize=letter) +styles = getSampleStyleSheet() +story = [] + +# Add content +title = Paragraph("Report Title", styles['Title']) +story.append(title) +story.append(Spacer(1, 12)) + +body = Paragraph("This is the body of the report. " * 20, styles['Normal']) +story.append(body) +story.append(PageBreak()) + +# Page 2 +story.append(Paragraph("Page 2", styles['Heading1'])) +story.append(Paragraph("Content for page 2", styles['Normal'])) + +# Build PDF +doc.build(story) +``` + +## Command-Line Tools + +### pdftotext (poppler-utils) +```bash +# Extract text +pdftotext input.pdf output.txt + +# Extract text preserving layout +pdftotext -layout input.pdf output.txt + +# Extract specific pages +pdftotext -f 1 -l 5 input.pdf output.txt # Pages 1-5 +``` + +### qpdf +```bash +# Merge PDFs +qpdf --empty --pages file1.pdf file2.pdf -- merged.pdf + +# Split pages +qpdf input.pdf --pages . 1-5 -- pages1-5.pdf +qpdf input.pdf --pages . 6-10 -- pages6-10.pdf + +# Rotate pages +qpdf input.pdf output.pdf --rotate=+90:1 # Rotate page 1 by 90 degrees + +# Remove password +qpdf --password=mypassword --decrypt encrypted.pdf decrypted.pdf +``` + +### pdftk (if available) +```bash +# Merge +pdftk file1.pdf file2.pdf cat output merged.pdf + +# Split +pdftk input.pdf burst + +# Rotate +pdftk input.pdf rotate 1east output rotated.pdf +``` + +## Common Tasks + +### Extract Text from Scanned PDFs +```python +# Requires: pip install pytesseract pdf2image +import pytesseract +from pdf2image import convert_from_path + +# Convert PDF to images +images = convert_from_path('scanned.pdf') + +# OCR each page +text = "" +for i, image in enumerate(images): + text += f"Page {i+1}:\n" + text += pytesseract.image_to_string(image) + text += "\n\n" + +print(text) +``` + +### Add Watermark +```python +from pypdf import PdfReader, PdfWriter + +# Create watermark (or load existing) +watermark = PdfReader("watermark.pdf").pages[0] + +# Apply to all pages +reader = PdfReader("document.pdf") +writer = PdfWriter() + +for page in reader.pages: + page.merge_page(watermark) + writer.add_page(page) + +with open("watermarked.pdf", "wb") as output: + writer.write(output) +``` + +### Extract Images +```bash +# Using pdfimages (poppler-utils) +pdfimages -j input.pdf output_prefix + +# This extracts all images as output_prefix-000.jpg, output_prefix-001.jpg, etc. +``` + +### Password Protection +```python +from pypdf import PdfReader, PdfWriter + +reader = PdfReader("input.pdf") +writer = PdfWriter() + +for page in reader.pages: + writer.add_page(page) + +# Add password +writer.encrypt("userpassword", "ownerpassword") + +with open("encrypted.pdf", "wb") as output: + writer.write(output) +``` + +## Quick Reference + +| Task | Best Tool | Command/Code | +|------|-----------|--------------| +| Merge PDFs | pypdf | `writer.add_page(page)` | +| Split PDFs | pypdf | One page per file | +| Extract text | pdfplumber | `page.extract_text()` | +| Extract tables | pdfplumber | `page.extract_tables()` | +| Create PDFs | reportlab | Canvas or Platypus | +| Command line merge | qpdf | `qpdf --empty --pages ...` | +| OCR scanned PDFs | pytesseract | Convert to image first | +| Fill PDF forms | pdf-lib or pypdf (see forms.md) | See forms.md | + +## Examples + +**Example 1: Extract tables from PDF report** +``` +User: "Pull the tables out of this quarterly report PDF" +→ Opens PDF with pdfplumber +→ Extracts tables, converts to pandas DataFrame +→ Exports to Excel file with clean formatting +``` + +**Example 2: Merge multiple PDFs** +``` +User: "Combine these three contracts into one PDF" +→ Uses pypdf to read all input files +→ Adds pages sequentially to new writer +→ Saves merged document to output path +``` + +**Example 3: Fill out a PDF form** +``` +User: "Fill in this tax form with my info" +→ Reads forms.md for form-filling workflow +→ Uses pdf-lib to populate form fields +→ Saves completed PDF with flattened form data +``` + +## Next Steps + +- For advanced pypdfium2 usage, see reference.md +- For JavaScript libraries (pdf-lib), see reference.md +- If you need to fill out a PDF form, follow the instructions in forms.md +- For troubleshooting guides, see reference.md diff --git a/.opencode/skills/Utilities/Pdf/Scripts/check_bounding_boxes.py b/.opencode/skills/Utilities/Pdf/Scripts/check_bounding_boxes.py new file mode 100755 index 00000000..7443660c --- /dev/null +++ b/.opencode/skills/Utilities/Pdf/Scripts/check_bounding_boxes.py @@ -0,0 +1,70 @@ +from dataclasses import dataclass +import json +import sys + + +# Script to check that the `fields.json` file that Claude creates when analyzing PDFs +# does not have overlapping bounding boxes. See forms.md. + + +@dataclass +class RectAndField: + rect: list[float] + rect_type: str + field: dict + + +# Returns a list of messages that are printed to stdout for Claude to read. +def get_bounding_box_messages(fields_json_stream) -> list[str]: + messages = [] + fields = json.load(fields_json_stream) + messages.append(f"Read {len(fields['form_fields'])} fields") + + def rects_intersect(r1, r2): + disjoint_horizontal = r1[0] >= r2[2] or r1[2] <= r2[0] + disjoint_vertical = r1[1] >= r2[3] or r1[3] <= r2[1] + return not (disjoint_horizontal or disjoint_vertical) + + rects_and_fields = [] + for f in fields["form_fields"]: + rects_and_fields.append(RectAndField(f["label_bounding_box"], "label", f)) + rects_and_fields.append(RectAndField(f["entry_bounding_box"], "entry", f)) + + has_error = False + for i, ri in enumerate(rects_and_fields): + # This is O(N^2); we can optimize if it becomes a problem. + for j in range(i + 1, len(rects_and_fields)): + rj = rects_and_fields[j] + if ri.field["page_number"] == rj.field["page_number"] and rects_intersect(ri.rect, rj.rect): + has_error = True + if ri.field is rj.field: + messages.append(f"FAILURE: intersection between label and entry bounding boxes for `{ri.field['description']}` ({ri.rect}, {rj.rect})") + else: + messages.append(f"FAILURE: intersection between {ri.rect_type} bounding box for `{ri.field['description']}` ({ri.rect}) and {rj.rect_type} bounding box for `{rj.field['description']}` ({rj.rect})") + if len(messages) >= 20: + messages.append("Aborting further checks; fix bounding boxes and try again") + return messages + if ri.rect_type == "entry": + if "entry_text" in ri.field: + font_size = ri.field["entry_text"].get("font_size", 14) + entry_height = ri.rect[3] - ri.rect[1] + if entry_height < font_size: + has_error = True + messages.append(f"FAILURE: entry bounding box height ({entry_height}) for `{ri.field['description']}` is too short for the text content (font size: {font_size}). Increase the box height or decrease the font size.") + if len(messages) >= 20: + messages.append("Aborting further checks; fix bounding boxes and try again") + return messages + + if not has_error: + messages.append("SUCCESS: All bounding boxes are valid") + return messages + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: check_bounding_boxes.py [fields.json]") + sys.exit(1) + # Input file should be in the `fields.json` format described in forms.md. + with open(sys.argv[1]) as f: + messages = get_bounding_box_messages(f) + for msg in messages: + print(msg) diff --git a/.opencode/skills/Utilities/Pdf/Scripts/check_bounding_boxes_test.py b/.opencode/skills/Utilities/Pdf/Scripts/check_bounding_boxes_test.py new file mode 100755 index 00000000..1dbb463c --- /dev/null +++ b/.opencode/skills/Utilities/Pdf/Scripts/check_bounding_boxes_test.py @@ -0,0 +1,226 @@ +import unittest +import json +import io +from check_bounding_boxes import get_bounding_box_messages + + +# Currently this is not run automatically in CI; it's just for documentation and manual checking. +class TestGetBoundingBoxMessages(unittest.TestCase): + + def create_json_stream(self, data): + """Helper to create a JSON stream from data""" + return io.StringIO(json.dumps(data)) + + def test_no_intersections(self): + """Test case with no bounding box intersections""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 30] + }, + { + "description": "Email", + "page_number": 1, + "label_bounding_box": [10, 40, 50, 60], + "entry_bounding_box": [60, 40, 150, 60] + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("SUCCESS" in msg for msg in messages)) + self.assertFalse(any("FAILURE" in msg for msg in messages)) + + def test_label_entry_intersection_same_field(self): + """Test intersection between label and entry of the same field""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 60, 30], + "entry_bounding_box": [50, 10, 150, 30] # Overlaps with label + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("FAILURE" in msg and "intersection" in msg for msg in messages)) + self.assertFalse(any("SUCCESS" in msg for msg in messages)) + + def test_intersection_between_different_fields(self): + """Test intersection between bounding boxes of different fields""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 30] + }, + { + "description": "Email", + "page_number": 1, + "label_bounding_box": [40, 20, 80, 40], # Overlaps with Name's boxes + "entry_bounding_box": [160, 10, 250, 30] + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("FAILURE" in msg and "intersection" in msg for msg in messages)) + self.assertFalse(any("SUCCESS" in msg for msg in messages)) + + def test_different_pages_no_intersection(self): + """Test that boxes on different pages don't count as intersecting""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 30] + }, + { + "description": "Email", + "page_number": 2, + "label_bounding_box": [10, 10, 50, 30], # Same coordinates but different page + "entry_bounding_box": [60, 10, 150, 30] + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("SUCCESS" in msg for msg in messages)) + self.assertFalse(any("FAILURE" in msg for msg in messages)) + + def test_entry_height_too_small(self): + """Test that entry box height is checked against font size""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 20], # Height is 10 + "entry_text": { + "font_size": 14 # Font size larger than height + } + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("FAILURE" in msg and "height" in msg for msg in messages)) + self.assertFalse(any("SUCCESS" in msg for msg in messages)) + + def test_entry_height_adequate(self): + """Test that adequate entry box height passes""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 30], # Height is 20 + "entry_text": { + "font_size": 14 # Font size smaller than height + } + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("SUCCESS" in msg for msg in messages)) + self.assertFalse(any("FAILURE" in msg for msg in messages)) + + def test_default_font_size(self): + """Test that default font size is used when not specified""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 20], # Height is 10 + "entry_text": {} # No font_size specified, should use default 14 + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("FAILURE" in msg and "height" in msg for msg in messages)) + self.assertFalse(any("SUCCESS" in msg for msg in messages)) + + def test_no_entry_text(self): + """Test that missing entry_text doesn't cause height check""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 20] # Small height but no entry_text + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("SUCCESS" in msg for msg in messages)) + self.assertFalse(any("FAILURE" in msg for msg in messages)) + + def test_multiple_errors_limit(self): + """Test that error messages are limited to prevent excessive output""" + fields = [] + # Create many overlapping fields + for i in range(25): + fields.append({ + "description": f"Field{i}", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], # All overlap + "entry_bounding_box": [20, 15, 60, 35] # All overlap + }) + + data = {"form_fields": fields} + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + # Should abort after ~20 messages + self.assertTrue(any("Aborting" in msg for msg in messages)) + # Should have some FAILURE messages but not hundreds + failure_count = sum(1 for msg in messages if "FAILURE" in msg) + self.assertGreater(failure_count, 0) + self.assertLess(len(messages), 30) # Should be limited + + def test_edge_touching_boxes(self): + """Test that boxes touching at edges don't count as intersecting""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [50, 10, 150, 30] # Touches at x=50 + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("SUCCESS" in msg for msg in messages)) + self.assertFalse(any("FAILURE" in msg for msg in messages)) + + +if __name__ == '__main__': + unittest.main() From a150931bd1c0f34a6de3a4b7b4005b19c7075b53 Mon Sep 17 00:00:00 2001 From: Steffen Zellmer <151627820+Steffen025@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:03:49 +0100 Subject: [PATCH 2/5] fix(pr07): address CodeRabbit Round 1 security findings Security fixes applied: - unpack.py: prevent zip-slip via safe extractall with path validation - document.py: fix import casing (ooxml.scripts -> Ooxml.Scripts) - BinaryTests.ts: prevent command injection by splitting command string to array - StateCheck.ts: prevent path traversal via resolve()+prefix check; mask env secrets in results - StaticAnalysis.ts: prevent command injection by splitting command string to array - SuiteManager.ts: validate suite name (alphanumeric/hyphens/underscores only); prevent silent overwrite of existing suites Config fix: - .coderabbit.yaml: change language from 'de' to 'en' for English reviews Intentionally skipped: - Browser/index.ts Dialog handle: architectural refactor, out of scope for this PR - BrowserSession.ts CORS: intentional local-dev-only setting - Cloudflare/Troubleshoot.md TypeScript in markdown: markdown doc, not runnable - Evals/AlgorithmBridge.ts synthetic executor: intentional design placeholder --- .coderabbit.yaml | 2 +- .../Utilities/Docx/Ooxml/Scripts/unpack.py | 9 +++++++- .../skills/Utilities/Docx/Scripts/document.py | 6 +++--- .../Evals/Graders/CodeBased/BinaryTests.ts | 3 ++- .../Evals/Graders/CodeBased/StateCheck.ts | 21 ++++++++++++++----- .../Evals/Graders/CodeBased/StaticAnalysis.ts | 3 ++- .../Utilities/Evals/Tools/SuiteManager.ts | 15 ++++++++++--- 7 files changed, 44 insertions(+), 15 deletions(-) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index dc1d4e32..8b47dc20 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -2,7 +2,7 @@ # Free Tier (OSS/personal project) # Docs: https://docs.coderabbit.ai/getting-started/configure-coderabbit -language: "de" +language: "en" # Review profile: assertive = flags everything, chill = only critical issues # For active AI development, "chill" prevents review noise on AI-generated code diff --git a/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py b/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py index 49387988..b0063d26 100755 --- a/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py +++ b/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py @@ -14,7 +14,14 @@ # Extract and format output_path = Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) -zipfile.ZipFile(input_file).extractall(output_path) + +# Safe extraction: prevent zip-slip by validating all member paths +with zipfile.ZipFile(input_file) as zf: + for member in zf.namelist(): + member_path = (output_path / member).resolve() + if not str(member_path).startswith(str(output_path.resolve())): + raise ValueError(f"Unsafe zip entry rejected (zip-slip): {member}") + zf.extractall(output_path) # Pretty print all XML files xml_files = list(output_path.rglob("*.xml")) + list(output_path.rglob("*.rels")) diff --git a/.opencode/skills/Utilities/Docx/Scripts/document.py b/.opencode/skills/Utilities/Docx/Scripts/document.py index ae9328dd..2b7a1660 100755 --- a/.opencode/skills/Utilities/Docx/Scripts/document.py +++ b/.opencode/skills/Utilities/Docx/Scripts/document.py @@ -34,9 +34,9 @@ from pathlib import Path from defusedxml import minidom -from ooxml.scripts.pack import pack_document -from ooxml.scripts.validation.docx import DOCXSchemaValidator -from ooxml.scripts.validation.redlining import RedliningValidator +from Ooxml.Scripts.pack import pack_document +from Ooxml.Scripts.validation.docx import DOCXSchemaValidator +from Ooxml.Scripts.validation.redlining import RedliningValidator from .utilities import XMLEditor diff --git a/.opencode/skills/Utilities/Evals/Graders/CodeBased/BinaryTests.ts b/.opencode/skills/Utilities/Evals/Graders/CodeBased/BinaryTests.ts index 8d69b92f..408cf0e0 100755 --- a/.opencode/skills/Utilities/Evals/Graders/CodeBased/BinaryTests.ts +++ b/.opencode/skills/Utilities/Evals/Graders/CodeBased/BinaryTests.ts @@ -30,7 +30,8 @@ export class BinaryTestsGrader extends BaseGrader { // Detect test command based on file extension const command = params.test_command ?? this.detectTestCommand(testFile); - const result = await $`cd ${workingDir} && timeout ${Math.ceil(timeout/1000)} ${command} ${testFile}` + const commandParts = command.split(/\s+/); + const result = await $`cd ${workingDir} && timeout ${Math.ceil(timeout/1000)} ${commandParts} ${testFile}` .quiet() .nothrow(); diff --git a/.opencode/skills/Utilities/Evals/Graders/CodeBased/StateCheck.ts b/.opencode/skills/Utilities/Evals/Graders/CodeBased/StateCheck.ts index d4dad447..5aafd0ab 100755 --- a/.opencode/skills/Utilities/Evals/Graders/CodeBased/StateCheck.ts +++ b/.opencode/skills/Utilities/Evals/Graders/CodeBased/StateCheck.ts @@ -6,7 +6,7 @@ import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts'; import type { GraderConfig, GraderResult, StateCheckParams } from '../../Types/index.ts'; import { existsSync, readFileSync } from 'fs'; -import { join } from 'path'; +import { join, resolve } from 'path'; export class StateCheckGrader extends BaseGrader { type = 'state_check' as const; @@ -35,7 +35,17 @@ export class StateCheckGrader extends BaseGrader { // Check file contents if (params.check_files) { for (const fileCheck of params.check_files) { - const filePath = join(workingDir, fileCheck.path); + const filePath = resolve(workingDir, fileCheck.path); + // Prevent path traversal: ensure resolved path stays within workingDir + if (!filePath.startsWith(resolve(workingDir) + '/') && filePath !== resolve(workingDir)) { + checks.push({ + check: `file.${fileCheck.path}`, + passed: false, + expected: 'file within working directory', + actual: 'path traversal rejected', + }); + continue; + } if (!existsSync(filePath)) { checks.push({ @@ -81,11 +91,12 @@ export class StateCheckGrader extends BaseGrader { if (params.check_env) { for (const [key, expected] of Object.entries(params.check_env)) { const actual = process.env[key]; + const passed = actual === expected; checks.push({ check: `env.${key}`, - passed: actual === expected, - expected, - actual, + passed, + expected: '***', // never expose expected secret value in results + actual: passed ? '*** (matches)' : '*** (mismatch)', }); } } diff --git a/.opencode/skills/Utilities/Evals/Graders/CodeBased/StaticAnalysis.ts b/.opencode/skills/Utilities/Evals/Graders/CodeBased/StaticAnalysis.ts index 2ef1f598..1162cc8b 100755 --- a/.opencode/skills/Utilities/Evals/Graders/CodeBased/StaticAnalysis.ts +++ b/.opencode/skills/Utilities/Evals/Graders/CodeBased/StaticAnalysis.ts @@ -26,7 +26,8 @@ export class StaticAnalysisGrader extends BaseGrader { for (const command of params.commands) { try { - const result = await $`cd ${workingDir} && ${command}`.quiet().nothrow(); + const commandParts = command.split(/\s+/); + const result = await $`cd ${workingDir} && ${commandParts}`.quiet().nothrow(); const output = result.stdout.toString() + result.stderr.toString(); const warnings = this.countIssues(output, 'warning'); diff --git a/.opencode/skills/Utilities/Evals/Tools/SuiteManager.ts b/.opencode/skills/Utilities/Evals/Tools/SuiteManager.ts index 14dd3326..62d90a7d 100755 --- a/.opencode/skills/Utilities/Evals/Tools/SuiteManager.ts +++ b/.opencode/skills/Utilities/Evals/Tools/SuiteManager.ts @@ -38,8 +38,20 @@ export function createSuite( tasks?: string[]; } ): EvalSuite { + // Validate name: alphanumeric, hyphens, underscores only — prevents path traversal + if (!/^[a-zA-Z0-9_-]+$/.test(name)) { + throw new Error(`Invalid suite name "${name}": only alphanumeric, hyphens, and underscores are allowed`); + } + ensureDirs(); + // Prevent accidental overwrites of existing suites + const dir = type === 'capability' ? 'Capability' : 'Regression'; + const filePath = join(SUITES_DIR, dir, `${name}.yaml`); + if (existsSync(filePath)) { + throw new Error(`Suite "${name}" already exists in ${dir}. Use a different name or delete the existing suite first.`); + } + const suite: EvalSuite = { name, description, @@ -51,9 +63,6 @@ export function createSuite( created_at: new Date().toISOString(), }; - const dir = type === 'capability' ? 'Capability' : 'Regression'; - const filePath = join(SUITES_DIR, dir, `${name}.yaml`); - writeFileSync(filePath, stringifyYaml(suite)); return suite; From ae8a56b7072677906f271f6900178fa52bc08042 Mon Sep 17 00:00:00 2001 From: Steffen Zellmer <151627820+Steffen025@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:52:05 +0100 Subject: [PATCH 3/5] fix(pr07): address CodeRabbit Round 2 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit unpack.py: - Replace assert with explicit if + sys.exit(1) for CLI validation (assert disabled under -O) - Replace startswith() zip-slip check with Path.relative_to() (not bypassable by prefix match) - Add noqa: S311 comment on random.choices RSID (local-only hint, not cryptographic) BinaryTests.ts: - Use results.length instead of params.test_files.length for score/passed calculation - Guard against zero-division: score = results.length ? passCount/results.length : 0 document.py: - Remove redundant local 'from datetime import datetime, timezone' inside _inject_attributes_to_nodes (already imported at module level) - Replace print(f'Using RSID: ...') with logger.debug() — library code must not write to stdout unconditionally Intentionally skipped: - BinaryTests.ts Bun.spawn rewrite: Bun $ with array interpolation is already injection-safe; full spawn rewrite is scope creep - document.py context manager (__enter__/__exit__/close()): architectural API addition, not a bug - BinaryTests.ts truncateMiddle helper: feature enhancement, not a bug --- .../skills/Utilities/Docx/Ooxml/Scripts/unpack.py | 11 ++++++++--- .opencode/skills/Utilities/Docx/Scripts/document.py | 7 ++++--- .../Utilities/Evals/Graders/CodeBased/BinaryTests.ts | 6 +++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py b/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py index b0063d26..abaef521 100755 --- a/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py +++ b/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py @@ -8,7 +8,9 @@ from pathlib import Path # Get command line arguments -assert len(sys.argv) == 3, "Usage: python unpack.py " +if len(sys.argv) != 3: + print("Usage: python unpack.py ", file=sys.stderr) + sys.exit(1) input_file, output_dir = sys.argv[1], sys.argv[2] # Extract and format @@ -16,10 +18,13 @@ output_path.mkdir(parents=True, exist_ok=True) # Safe extraction: prevent zip-slip by validating all member paths +out_resolved = output_path.resolve() with zipfile.ZipFile(input_file) as zf: for member in zf.namelist(): member_path = (output_path / member).resolve() - if not str(member_path).startswith(str(output_path.resolve())): + try: + member_path.relative_to(out_resolved) + except ValueError: raise ValueError(f"Unsafe zip entry rejected (zip-slip): {member}") zf.extractall(output_path) @@ -32,5 +37,5 @@ # For .docx files, suggest an RSID for tracked changes if input_file.endswith(".docx"): - suggested_rsid = "".join(random.choices("0123456789ABCDEF", k=8)) + suggested_rsid = "".join(random.choices("0123456789ABCDEF", k=8)) # noqa: S311 — local-only, non-cryptographic RSID for user hinting, not stored or reused print(f"Suggested RSID for edit session: {suggested_rsid}") diff --git a/.opencode/skills/Utilities/Docx/Scripts/document.py b/.opencode/skills/Utilities/Docx/Scripts/document.py index 2b7a1660..2f008790 100755 --- a/.opencode/skills/Utilities/Docx/Scripts/document.py +++ b/.opencode/skills/Utilities/Docx/Scripts/document.py @@ -27,12 +27,15 @@ """ import html +import logging import random import shutil import tempfile from datetime import datetime, timezone from pathlib import Path +logger = logging.getLogger(__name__) + from defusedxml import minidom from Ooxml.Scripts.pack import pack_document from Ooxml.Scripts.validation.docx import DOCXSchemaValidator @@ -127,8 +130,6 @@ def _inject_attributes_to_nodes(self, nodes): Args: nodes: List of DOM nodes to process """ - from datetime import datetime, timezone - timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def is_inside_deletion(elem): @@ -649,7 +650,7 @@ def __init__( # Generate RSID if not provided self.rsid = rsid if rsid else _generate_rsid() - print(f"Using RSID: {self.rsid}") + logger.debug("Using RSID: %s", self.rsid) # Set default author and initials self.author = author diff --git a/.opencode/skills/Utilities/Evals/Graders/CodeBased/BinaryTests.ts b/.opencode/skills/Utilities/Evals/Graders/CodeBased/BinaryTests.ts index 408cf0e0..73b04583 100755 --- a/.opencode/skills/Utilities/Evals/Graders/CodeBased/BinaryTests.ts +++ b/.opencode/skills/Utilities/Evals/Graders/CodeBased/BinaryTests.ts @@ -53,11 +53,11 @@ export class BinaryTestsGrader extends BaseGrader { } const passCount = results.filter(r => r.passed).length; - const score = passCount / params.test_files.length; - const passed = passCount === params.test_files.length; + const score = results.length ? passCount / results.length : 0; + const passed = results.length > 0 && passCount === results.length; return this.createResult(score, passed, performance.now() - start, { - reasoning: `${passCount}/${params.test_files.length} tests passed`, + reasoning: `${passCount}/${results.length} tests passed`, details: { results, working_dir: workingDir, From 9dcd7d5388fa33180fc0453f431936849594df5c Mon Sep 17 00:00:00 2001 From: Steffen Zellmer <151627820+Steffen025@users.noreply.github.com> Date: Mon, 16 Mar 2026 11:13:26 +0100 Subject: [PATCH 4/5] fix(pr07): address CodeRabbit Round 3 findings unpack.py: - Add exception chaining (raise ... from err) to preserve original ValueError context document.py: - __getitem__: validate xml_path stays within unpacked_path using resolve()+relative_to() - _ensure_comment_relationships: replace single early-return with per-item checks for each companion relationship; re-query get_next_rid() per item - _ensure_comment_content_types: replace single early-return with per-item checks for each companion content type - Replace default author 'Claude' with 'OpenCode' and initials 'C' with 'O' in both DocxXMLEditor and Document constructors (branding migration) Intentionally skipped: - Bun.$ array interpolation rewrite to Bun.spawn (not a genuine injection vector) - Mass code-fence language tagging of reference files (scope creep) --- .../Utilities/Docx/Ooxml/Scripts/unpack.py | 4 +- .../skills/Utilities/Docx/Scripts/document.py | 62 ++++++++++--------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py b/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py index abaef521..53b98b7e 100755 --- a/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py +++ b/.opencode/skills/Utilities/Docx/Ooxml/Scripts/unpack.py @@ -24,8 +24,8 @@ member_path = (output_path / member).resolve() try: member_path.relative_to(out_resolved) - except ValueError: - raise ValueError(f"Unsafe zip entry rejected (zip-slip): {member}") + except ValueError as err: + raise ValueError(f"Unsafe zip entry rejected (zip-slip): {member}") from err zf.extractall(output_path) # Pretty print all XML files diff --git a/.opencode/skills/Utilities/Docx/Scripts/document.py b/.opencode/skills/Utilities/Docx/Scripts/document.py index 2f008790..ec8e2db7 100755 --- a/.opencode/skills/Utilities/Docx/Scripts/document.py +++ b/.opencode/skills/Utilities/Docx/Scripts/document.py @@ -60,15 +60,15 @@ class DocxXMLEditor(XMLEditor): """ def __init__( - self, xml_path, rsid: str, author: str = "Claude", initials: str = "C" + self, xml_path, rsid: str, author: str = "OpenCode", initials: str = "O" ): """Initialize with required RSID and optional author. Args: xml_path: Path to XML file to edit rsid: RSID to automatically apply to new elements - author: Author name for tracked changes and comments (default: "Claude") - initials: Author initials (default: "C") + author: Author name for tracked changes and comments (default: "OpenCode") + initials: Author initials (default: "O") """ super().__init__(xml_path) self.rsid = rsid @@ -618,8 +618,8 @@ def __init__( unpacked_dir, rsid=None, track_revisions=False, - author="Claude", - initials="C", + author="OpenCode", + initials="O", ): """ Initialize with path to unpacked Word document directory. @@ -629,8 +629,8 @@ def __init__( unpacked_dir: Path to unpacked DOCX directory (must contain word/ subdirectory) rsid: Optional RSID to use for all comment elements. If not provided, one will be generated. track_revisions: If True, enables track revisions in settings.xml (default: False) - author: Default author name for comments (default: "Claude") - initials: Default author initials for comments (default: "C") + author: Default author name for comments (default: "OpenCode") + initials: Default author initials for comments (default: "O") """ self.original_path = Path(unpacked_dir) @@ -703,6 +703,11 @@ def __getitem__(self, xml_path: str) -> DocxXMLEditor: """ if xml_path not in self._editors: file_path = self.unpacked_path / xml_path + # Validate that the resolved path stays within the unpacked directory + try: + file_path.resolve().relative_to(self.unpacked_path.resolve()) + except ValueError: + raise ValueError(f"Path traversal detected: {xml_path}") if not file_path.exists(): raise ValueError(f"XML file not found: {xml_path}") # Use DocxXMLEditor with RSID, author, and initials for all editors @@ -1205,53 +1210,49 @@ def _ensure_comment_relationships(self): """Ensure word/_rels/document.xml.rels has comment relationships.""" editor = self["word/_rels/document.xml.rels"] - if self._has_relationship(editor, "comments.xml"): - return - root = editor.dom.documentElement root_tag = root.tagName # type: ignore prefix = root_tag.split(":")[0] + ":" if ":" in root_tag else "" - next_rid_num = int(editor.get_next_rid()[3:]) - # Add relationship elements - rels = [ + # Check and add each relationship individually so that a pre-existing + # comments.xml relationship does not prevent companion relationships + # (commentsExtended, commentsIds, commentsExtensible) from being added. + rels_to_ensure = [ ( - next_rid_num, "http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments", "comments.xml", ), ( - next_rid_num + 1, "http://schemas.microsoft.com/office/2011/relationships/commentsExtended", "commentsExtended.xml", ), ( - next_rid_num + 2, "http://schemas.microsoft.com/office/2016/09/relationships/commentsIds", "commentsIds.xml", ), ( - next_rid_num + 3, "http://schemas.microsoft.com/office/2018/08/relationships/commentsExtensible", "commentsExtensible.xml", ), ] - for rel_id, rel_type, target in rels: - rel_xml = f'<{prefix}Relationship Id="rId{rel_id}" Type="{rel_type}" Target="{target}"/>' - editor.append_to(root, rel_xml) + for rel_type, target in rels_to_ensure: + if not self._has_relationship(editor, target): + # Re-query next rId each time since prior iterations may have added one + next_rid_num = int(editor.get_next_rid()[3:]) + rel_xml = f'<{prefix}Relationship Id="rId{next_rid_num}" Type="{rel_type}" Target="{target}"/>' + editor.append_to(root, rel_xml) def _ensure_comment_content_types(self): """Ensure [Content_Types].xml has comment content types.""" editor = self["[Content_Types].xml"] - if self._has_override(editor, "/word/comments.xml"): - return - root = editor.dom.documentElement - # Add Override elements - overrides = [ + # Check and add each Override individually so that a pre-existing + # comments.xml entry does not prevent companion content types + # (commentsExtended, commentsIds, commentsExtensible) from being added. + overrides_to_ensure = [ ( "/word/comments.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml", @@ -1270,8 +1271,9 @@ def _ensure_comment_content_types(self): ), ] - for part_name, content_type in overrides: - override_xml = ( - f'' - ) - editor.append_to(root, override_xml) + for part_name, content_type in overrides_to_ensure: + if not self._has_override(editor, part_name): + override_xml = ( + f'' + ) + editor.append_to(root, override_xml) From 0e8dee8e631050303a40a1b860e890b68e7971b6 Mon Sep 17 00:00:00 2001 From: Steffen Zellmer <151627820+Steffen025@users.noreply.github.com> Date: Mon, 16 Mar 2026 11:37:39 +0100 Subject: [PATCH 5/5] fix(pr07): address CodeRabbit Round 4 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit document.py: - _load_existing_comments: wrap int(comment_id) in try/except ValueError and skip non-negative-integer IDs (mirrors _get_next_comment_id pattern) - Document.__init__: add fast-fail guard after self.word_path assignment — raise ValueError if 'word' directory is missing or not a directory - __getitem__: normalize cache key using resolve()+relative_to() so aliased paths (e.g. 'word/../word/document.xml') reuse the same editor instance; traversal detection is now a side-effect of the same normalization step --- .../skills/Utilities/Docx/Scripts/document.py | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/.opencode/skills/Utilities/Docx/Scripts/document.py b/.opencode/skills/Utilities/Docx/Scripts/document.py index ec8e2db7..4e15895d 100755 --- a/.opencode/skills/Utilities/Docx/Scripts/document.py +++ b/.opencode/skills/Utilities/Docx/Scripts/document.py @@ -647,6 +647,10 @@ def __init__( pack_document(self.original_path, self.original_docx, validate=False) self.word_path = self.unpacked_path / "word" + if not self.word_path.exists() or not self.word_path.is_dir(): + raise ValueError( + "Missing required 'word' directory in DOCX unpacked content" + ) # Generate RSID if not provided self.rsid = rsid if rsid else _generate_rsid() @@ -701,20 +705,22 @@ def __getitem__(self, xml_path: str) -> DocxXMLEditor: # Get node from comments.xml comment = doc["word/comments.xml"].get_node(tag="w:comment", attrs={"w:id": "0"}) """ - if xml_path not in self._editors: - file_path = self.unpacked_path / xml_path - # Validate that the resolved path stays within the unpacked directory - try: + # Normalize the key to prevent aliased paths creating duplicate editors + file_path = self.unpacked_path / xml_path + try: + normalized_key = str( file_path.resolve().relative_to(self.unpacked_path.resolve()) - except ValueError: - raise ValueError(f"Path traversal detected: {xml_path}") + ) + except ValueError: + raise ValueError(f"Path traversal detected: {xml_path}") + if normalized_key not in self._editors: if not file_path.exists(): raise ValueError(f"XML file not found: {xml_path}") # Use DocxXMLEditor with RSID, author, and initials for all editors - self._editors[xml_path] = DocxXMLEditor( + self._editors[normalized_key] = DocxXMLEditor( file_path, rsid=self.rsid, author=self.author, initials=self.initials ) - return self._editors[xml_path] + return self._editors[normalized_key] def add_comment(self, start, end, text: str) -> int: """ @@ -920,6 +926,14 @@ def _load_existing_comments(self): if not comment_id: continue + # Defensively parse comment_id (mirrors _get_next_comment_id) + try: + comment_id_int = int(comment_id) + except ValueError: + continue + if comment_id_int < 0: + continue + # Find para_id from the w:p element within the comment para_id = None for p_elem in comment_elem.getElementsByTagName("w:p"): @@ -930,7 +944,7 @@ def _load_existing_comments(self): if not para_id: continue - existing[int(comment_id)] = {"para_id": para_id} + existing[comment_id_int] = {"para_id": para_id} return existing