Enhance NotebookLM integration and frontend video ingestion pipeline #51
The diff you're trying to view is too large. We only load the first 3000 changed files.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| {"schema":["../dataconnect/.dataconnect/**/*.gql","../dataconnect/schema/**/*.gql"],"document":["../dataconnect/example/**/*.gql"]} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| { | ||
| "projects": { | ||
| "default": "uvai-730bb" | ||
| }, | ||
| "targets": {}, | ||
| "etags": {} | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -155,3 +155,4 @@ UVAI_Digital_Refinery_Blueprint.pdf | |
| *.db | ||
| .vercel | ||
| .env*.local | ||
| .next/ | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
| @@ -1,20 +1,15 @@ | ||||||||
| import OpenAI from 'openai'; | ||||||||
| import { Type } from '@google/genai'; | ||||||||
| import { NextResponse } from 'next/server'; | ||||||||
| import { getGeminiClient, hasGeminiKey } from '@/lib/gemini-client'; | ||||||||
|
|
||||||||
| let _openai: OpenAI | null = null; | ||||||||
| function getOpenAI() { | ||||||||
| if (!_openai) _openai = new OpenAI(); | ||||||||
| return _openai; | ||||||||
| } | ||||||||
|
|
||||||||
| // JSON Schema for structured extraction via OpenAI Responses API | ||||||||
| const extractionSchema = { | ||||||||
| type: 'object' as const, | ||||||||
| properties: { | ||||||||
|
|
@@ -54,6 +49,43 @@ const extractionSchema = { | |||||||
| additionalProperties: false, | ||||||||
| }; | ||||||||
|
|
||||||||
| // Gemini responseSchema using @google/genai Type system | ||||||||
| const geminiResponseSchema = { | ||||||||
| type: Type.OBJECT, | ||||||||
| properties: { | ||||||||
| events: { | ||||||||
| type: Type.ARRAY, | ||||||||
| items: { | ||||||||
| type: Type.OBJECT, | ||||||||
| properties: { | ||||||||
| type: { type: Type.STRING, enum: ['action', 'topic', 'insight', 'tool', 'resource'] }, | ||||||||
| title: { type: Type.STRING }, | ||||||||
| description: { type: Type.STRING }, | ||||||||
| timestamp: { type: Type.STRING, nullable: true }, | ||||||||
| priority: { type: Type.STRING, enum: ['high', 'medium', 'low'] }, | ||||||||
| }, | ||||||||
| required: ['type', 'title', 'description', 'priority'], | ||||||||
| }, | ||||||||
| }, | ||||||||
| actions: { | ||||||||
| type: Type.ARRAY, | ||||||||
| items: { | ||||||||
| type: Type.OBJECT, | ||||||||
| properties: { | ||||||||
| title: { type: Type.STRING }, | ||||||||
| description: { type: Type.STRING }, | ||||||||
| category: { type: Type.STRING, enum: ['setup', 'build', 'deploy', 'learn', 'research', 'configure'] }, | ||||||||
| estimatedMinutes: { type: Type.NUMBER, nullable: true }, | ||||||||
| }, | ||||||||
| required: ['title', 'description', 'category'], | ||||||||
| }, | ||||||||
| }, | ||||||||
| summary: { type: Type.STRING }, | ||||||||
| topics: { type: Type.ARRAY, items: { type: Type.STRING } }, | ||||||||
| }, | ||||||||
| required: ['events', 'actions', 'summary', 'topics'], | ||||||||
| }; | ||||||||
|
|
||||||||
| const SYSTEM_PROMPT = `You are an expert content analyst. Extract structured data from video transcripts. | ||||||||
| Be specific and practical — no vague or generic items. | ||||||||
| For events: classify type (action/topic/insight/tool/resource) and priority (high/medium/low). | ||||||||
|
|
@@ -94,54 +126,91 @@ async function extractWithOpenAI(trimmed: string, videoTitle?: string, videoUrl? | |||||||
| } | ||||||||
|
|
||||||||
| async function extractWithGemini(trimmed: string, videoTitle?: string, videoUrl?: string) { | ||||||||
| const ai = getGeminiClient(); | ||||||||
| const response = await ai.models.generateContent({ | ||||||||
| model: 'gemini-3-pro-preview', | ||||||||
| contents: `${SYSTEM_PROMPT}\n\n${buildUserPrompt(trimmed, videoTitle, videoUrl)}`, | ||||||||
| config: { | ||||||||
| temperature: 0.3, | ||||||||
| responseMimeType: 'application/json', | ||||||||
| responseSchema: geminiResponseSchema, | ||||||||
| tools: [{ googleSearch: {} }], | ||||||||
| }, | ||||||||
| }); | ||||||||
| const text = response.text ?? ''; | ||||||||
| return JSON.parse(text); | ||||||||
| } | ||||||||
|
|
||||||||
| export async function POST(request: Request) { | ||||||||
| try { | ||||||||
| const { transcript, videoTitle, videoUrl } = await request.json(); | ||||||||
|
|
||||||||
| // Accept either transcript text OR videoUrl for direct Gemini analysis | ||||||||
| if ((!transcript || typeof transcript !== 'string') && !videoUrl) { | ||||||||
| return NextResponse.json( | ||||||||
| { error: 'transcript (string) or videoUrl is required' }, | ||||||||
| { status: 400 } | ||||||||
| ); | ||||||||
| } | ||||||||
|
|
||||||||
| let parsed; | ||||||||
| let provider = 'openai'; | ||||||||
|
|
||||||||
| // If we have transcript text, use the existing extraction logic | ||||||||
| if (transcript && typeof transcript === 'string' && transcript.length > 50) { | ||||||||
| const trimmed = transcript.slice(0, 8000); | ||||||||
|
|
||||||||
| if (process.env.OPENAI_API_KEY) { | ||||||||
| try { | ||||||||
| parsed = await extractWithOpenAI(trimmed, videoTitle, videoUrl); | ||||||||
| } catch (err) { | ||||||||
| const msg = err instanceof Error ? err.message : ''; | ||||||||
| if ((msg.includes('429') || msg.includes('quota') || msg.includes('rate')) && hasGeminiKey()) { | ||||||||
| console.warn('OpenAI quota hit, falling back to Gemini'); | ||||||||
| parsed = await extractWithGemini(trimmed, videoTitle, videoUrl); | ||||||||
| provider = 'gemini'; | ||||||||
| } else { | ||||||||
| throw err; | ||||||||
| } | ||||||||
| } | ||||||||
| } else if (hasGeminiKey()) { | ||||||||
| parsed = await extractWithGemini(trimmed, videoTitle, videoUrl); | ||||||||
| provider = 'gemini'; | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| // If no transcript but have videoUrl + Gemini, do direct video analysis via Google Search | ||||||||
| if (!parsed && videoUrl && hasGeminiKey()) { | ||||||||
| try { | ||||||||
| const ai = getGeminiClient(); | ||||||||
| const response = await ai.models.generateContent({ | ||||||||
| model: 'gemini-3-pro-preview', | ||||||||
| contents: `${SYSTEM_PROMPT}\n\nAnalyze this YouTube video and extract structured data. | ||||||||
| Use your Google Search tool to find the video's transcript, description, and chapter content. | ||||||||
|
|
||||||||
| Video URL: ${videoUrl} | ||||||||
| ${videoTitle ? `Video Title: ${videoTitle}` : ''} | ||||||||
|
|
||||||||
| Extract events, actions, summary, and topics from the actual video content found via search.`, | ||||||||
| config: { | ||||||||
| temperature: 0.3, | ||||||||
| responseMimeType: 'application/json', | ||||||||
| responseSchema: geminiResponseSchema, | ||||||||
| tools: [{ googleSearch: {} }], | ||||||||
| }, | ||||||||
| }); | ||||||||
| const text = response.text ?? ''; | ||||||||
| parsed = JSON.parse(text); | ||||||||
| provider = 'gemini-search'; | ||||||||
| } catch (e) { | ||||||||
| console.warn('Gemini direct video extraction failed:', e); | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| if (!parsed) { | ||||||||
| return NextResponse.json({ | ||||||||
| success: false, | ||||||||
| error: 'No AI API key configured or all extraction attempts failed. Set GEMINI_API_KEY.', | ||||||||
|
There was a problem hiding this comment. This error message is misleading because extraction can succeed with
Suggested change
|
||||||||
| data: { events: [], actions: [], summary: '', topics: [] }, | ||||||||
| }); | ||||||||
| } | ||||||||
|
|
||||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,163 @@ | ||||||||
| import { NextResponse } from 'next/server'; | ||||||||
| import { publishEvent, EventTypes } from '@/lib/cloudevents'; | ||||||||
| import { analyzeVideoWithGemini } from '@/lib/gemini-video-analyzer'; | ||||||||
| import { hasGeminiKey } from '@/lib/gemini-client'; | ||||||||
|
|
||||||||
| const rawBackendUrl = process.env.BACKEND_URL || ''; | ||||||||
| const BACKEND_URL = rawBackendUrl.startsWith('http') ? rawBackendUrl : 'http://localhost:8000'; | ||||||||
| const BACKEND_AVAILABLE = rawBackendUrl.startsWith('http'); | ||||||||
|
|
||||||||
| /** | ||||||||
| * POST /api/pipeline | ||||||||
| * | ||||||||
| * End-to-end pipeline: YouTube URL → Video Analysis → Code Generation → Deployment → Live URL | ||||||||
| * | ||||||||
| * This is the FULL pipeline that the user's notes describe (PK=999, PK=1021): | ||||||||
| * Ingest → Translate → Transport → Execute | ||||||||
| * | ||||||||
| * Strategies: | ||||||||
| * 1. Backend pipeline (FastAPI /api/v1/video-to-software) — full pipeline with agents | ||||||||
| * 2. Gemini analysis + frontend deployment — when no backend is available | ||||||||
| */ | ||||||||
| export async function POST(request: Request) { | ||||||||
| let videoUrl: string | undefined; | ||||||||
| try { | ||||||||
| const body = await request.json(); | ||||||||
| const { url, project_type = 'web', deployment_target = 'vercel', features } = body; | ||||||||
| videoUrl = url; | ||||||||
|
|
||||||||
| if (!url) { | ||||||||
| return NextResponse.json({ error: 'Video URL is required' }, { status: 400 }); | ||||||||
| } | ||||||||
|
|
||||||||
| await publishEvent(EventTypes.VIDEO_RECEIVED, { url, pipeline: 'end-to-end' }, url); | ||||||||
|
|
||||||||
| // ── Strategy 1: Full backend pipeline (FastAPI video-to-software) ── | ||||||||
| if (BACKEND_AVAILABLE) { | ||||||||
| try { | ||||||||
| const controller = new AbortController(); | ||||||||
| const timeout = setTimeout(() => controller.abort(), 300_000); // 5 min for full pipeline | ||||||||
|
|
||||||||
| let response: Response; | ||||||||
| try { | ||||||||
| response = await fetch(`${BACKEND_URL}/api/v1/video-to-software`, { | ||||||||
| method: 'POST', | ||||||||
| headers: { 'Content-Type': 'application/json' }, | ||||||||
| body: JSON.stringify({ | ||||||||
| video_url: url, | ||||||||
| project_type, | ||||||||
| deployment_target, | ||||||||
| features: features || ['responsive_design', 'modern_ui'], | ||||||||
| }), | ||||||||
| signal: controller.signal, | ||||||||
| }); | ||||||||
| } finally { | ||||||||
| clearTimeout(timeout); | ||||||||
| } | ||||||||
|
|
||||||||
| if (response.ok) { | ||||||||
| const result = await response.json(); | ||||||||
|
|
||||||||
| await publishEvent(EventTypes.PIPELINE_COMPLETED, { | ||||||||
| strategy: 'backend-pipeline', | ||||||||
| success: result.status === 'success', | ||||||||
| live_url: result.live_url, | ||||||||
| github_repo: result.github_repo, | ||||||||
| build_status: result.build_status, | ||||||||
| }, url); | ||||||||
|
|
||||||||
| return NextResponse.json({ | ||||||||
| id: `pipeline_${Date.now().toString(36)}`, | ||||||||
|
Contributor
There was a problem hiding this comment. Using
Suggested change
|
||||||||
| status: result.status || 'complete', | ||||||||
| pipeline: 'backend', | ||||||||
| processing_time: result.processing_time, | ||||||||
| result: { | ||||||||
| live_url: result.live_url, | ||||||||
| github_repo: result.github_repo, | ||||||||
| build_status: result.build_status, | ||||||||
| video_analysis: result.video_analysis, | ||||||||
| code_generation: result.code_generation, | ||||||||
| deployment: result.deployment, | ||||||||
| features_implemented: result.features_implemented, | ||||||||
| }, | ||||||||
| }); | ||||||||
| } | ||||||||
| console.warn(`Backend pipeline returned ${response.status}, falling back`); | ||||||||
| } catch (e) { | ||||||||
| console.log('Backend pipeline unavailable:', e); | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| // ── Strategy 2: Gemini analysis (video intelligence only, no deployment) ── | ||||||||
| if (hasGeminiKey()) { | ||||||||
| try { | ||||||||
| const startTime = Date.now(); | ||||||||
| const analysis = await analyzeVideoWithGemini(url); | ||||||||
| const elapsed = Date.now() - startTime; | ||||||||
|
|
||||||||
| await publishEvent(EventTypes.PIPELINE_COMPLETED, { | ||||||||
| strategy: 'gemini-analysis-only', | ||||||||
| success: true, | ||||||||
| note: 'Backend unavailable — analysis only, no deployment', | ||||||||
| }, url); | ||||||||
|
|
||||||||
| return NextResponse.json({ | ||||||||
| id: `pipeline_${Date.now().toString(36)}`, | ||||||||
|
Contributor
There was a problem hiding this comment. Using
Suggested change
|
||||||||
| status: 'partial', | ||||||||
| pipeline: 'gemini-only', | ||||||||
| processing_time: `${(elapsed / 1000).toFixed(1)}s`, | ||||||||
| result: { | ||||||||
| live_url: null, | ||||||||
| github_repo: null, | ||||||||
| build_status: 'not_attempted', | ||||||||
| video_analysis: { | ||||||||
| title: analysis.title, | ||||||||
| summary: analysis.summary, | ||||||||
| events: analysis.events, | ||||||||
| actions: analysis.actions, | ||||||||
| topics: analysis.topics, | ||||||||
| architectureCode: analysis.architectureCode, | ||||||||
| }, | ||||||||
| code_generation: null, | ||||||||
| deployment: null, | ||||||||
| message: 'Backend pipeline unavailable. Video analysis complete but code generation and deployment require the Python backend.', | ||||||||
| }, | ||||||||
| }); | ||||||||
| } catch (e) { | ||||||||
| console.error('Gemini analysis failed:', e); | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| return NextResponse.json( | ||||||||
| { error: 'No pipeline available. Configure BACKEND_URL for full pipeline or GEMINI_API_KEY for analysis only.' }, | ||||||||
| { status: 503 }, | ||||||||
| ); | ||||||||
| } catch (error) { | ||||||||
| console.error('Pipeline error:', error); | ||||||||
| await publishEvent(EventTypes.PIPELINE_FAILED, { error: String(error) }, videoUrl).catch(() => {}); | ||||||||
| return NextResponse.json( | ||||||||
| { error: 'Pipeline failed', details: String(error) }, | ||||||||
| { status: 500 }, | ||||||||
| ); | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| export async function GET() { | ||||||||
| return NextResponse.json({ | ||||||||
| name: 'EventRelay End-to-End Pipeline', | ||||||||
| version: '1.0.0', | ||||||||
| description: 'YouTube URL → Video Analysis → Code Generation → Deployment → Live URL', | ||||||||
| pipeline_stages: [ | ||||||||
| '1. Ingest: Gemini analyzes video content with Google Search grounding', | ||||||||
| '2. Translate: Structured output → VideoPack artifact', | ||||||||
| '3. Transport: CloudEvents published at each stage', | ||||||||
| '4. Execute: Agents generate code, create repo, deploy to Vercel', | ||||||||
| ], | ||||||||
| backend_available: BACKEND_AVAILABLE, | ||||||||
| gemini_available: hasGeminiKey(), | ||||||||
| endpoints: { | ||||||||
| pipeline: 'POST /api/pipeline - Full end-to-end pipeline', | ||||||||
| video: 'POST /api/video - Video analysis only', | ||||||||
| }, | ||||||||
| }); | ||||||||
| } | ||||||||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This direct call to
JSON.parse()is unsafe. Ifresponse.textfrom the Gemini API is an empty string (which is possible if the model returns no content), this will throw an unhandled exception and cause the API route to crash with a 500 error. You should gracefully handle the case of an empty or invalid JSON string before parsing.