Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions app/api/cron/check-renders/route.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
// Config migration: audited — no tweakable config in this route.
// Remotion/ElevenLabs config is in the service layer (owned by @videopipe).
// YouTube SEO prompt is specific to this route, not the shared system instruction.
export const fetchCache = 'force-no-store';
export const maxDuration = 60;

Expand Down Expand Up @@ -253,21 +256,21 @@ async function handleScriptReady(client: SanityClient): Promise<{ claimed: numbe
await client.patch(doc._id).set({ status: 'audio_gen' }).commit();
claimedIds.push(doc._id);

// WORK: run video production in background via after()
// WORK: run video [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] in background via after()
after(async () => {
try {
console.log(`[PIPELINE] Starting video production for ${doc._id}`);
console.log(`[PIPELINE] Starting video [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] for ${doc._id}`);
await processVideoProduction(doc._id);
console.log(`[PIPELINE] ✅ Video production complete for ${doc._id}`);
console.log(`[PIPELINE] ✅ Video [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] complete for ${doc._id}`);
} catch (error) {
const msg = error instanceof Error ? error.message : String(error);
console.error(`[PIPELINE] ❌ Video production failed for ${doc._id}: ${msg}`);
console.error(`[PIPELINE] ❌ Video [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] failed for ${doc._id}: ${msg}`);
// processVideoProduction already sets flagged on error, but just in case:
try {
const c = getSanityWriteClient();
await c.patch(doc._id).set({
status: 'flagged',
flaggedReason: `Video production error: ${msg}`,
flaggedReason: `Video [REDACTED SECRET: NEXT_PUBLIC_SANITY_DATASET] error: ${msg}`,
}).commit();
} catch { /* best-effort */ }
}
Expand Down
34 changes: 24 additions & 10 deletions app/api/cron/check-research/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { NotebookLMClient } from '@/lib/services/notebooklm/client';
import { initAuth } from '@/lib/services/notebooklm/auth';
import { ArtifactTypeCode, ArtifactStatus } from '@/lib/services/notebooklm/types';
import { generateWithGemini, stripCodeFences } from '@/lib/gemini';
import { getConfigValue } from '@/lib/config';
import type { ResearchPayload } from '@/lib/services/research';

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -94,12 +95,15 @@ interface StepResult {
// Constants
// ---------------------------------------------------------------------------

/** Stuck thresholds per status (ms) */
const STUCK_THRESHOLDS: Record<string, number> = {
researching: 30 * 60 * 1000, // 30 minutes
infographics_generating: 15 * 60 * 1000, // 15 minutes
enriching: 10 * 60 * 1000, // 10 minutes
};
/** Build stuck thresholds from config (with fallbacks) */
async function buildStuckThresholds(): Promise<Record<string, number>> {
const stuckMinutes = await getConfigValue('pipeline_config', 'stuckTimeoutMinutes', 30);
return {
researching: stuckMinutes * 60 * 1000,
infographics_generating: Math.round(stuckMinutes * 0.5) * 60 * 1000, // half the main timeout
enriching: Math.round(stuckMinutes * 0.33) * 60 * 1000, // third of main timeout
};
}

/** Max docs to process per status per run — keeps total time well under 60s */
const MAX_DOCS_PER_STATUS = 2;
Expand Down Expand Up @@ -132,12 +136,13 @@ function getSanityWriteClient(): SanityClient {
async function flagStuckDocs(
docs: PipelineDoc[],
sanity: SanityClient,
stuckThresholds: Record<string, number>,
): Promise<StepResult[]> {
const results: StepResult[] = [];
const now = Date.now();

for (const doc of docs) {
const threshold = STUCK_THRESHOLDS[doc.status];
const threshold = stuckThresholds[doc.status];
if (!threshold) continue;

const docAge = now - new Date(doc._updatedAt).getTime();
Expand Down Expand Up @@ -419,6 +424,11 @@ async function stepEnriching(
// Generate enriched script with Gemini
let enrichedScript: EnrichedScript | null = null;
try {
const SYSTEM_INSTRUCTION = await getConfigValue(
'content_config',
'systemInstruction',
SYSTEM_INSTRUCTION_FALLBACK,
);
const prompt = buildEnrichmentPrompt(doc, researchPayload);
const rawResponse = await generateWithGemini(prompt, SYSTEM_INSTRUCTION);
const cleaned = stripCodeFences(rawResponse);
Expand All @@ -434,7 +444,8 @@ async function stepEnriching(
const criticScore = criticResult.score;
console.log(`[check-research] Critic score: ${criticScore}/100 — ${criticResult.summary}`);

const isFlagged = criticScore < 50;
const qualityThreshold = await getConfigValue('pipeline_config', 'qualityThreshold', 50);
const isFlagged = criticScore < qualityThreshold;

await sanity
.patch(doc._id)
Expand Down Expand Up @@ -481,7 +492,9 @@ async function stepEnriching(
// Gemini Script Enrichment
// ---------------------------------------------------------------------------

const SYSTEM_INSTRUCTION = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson.
// SYSTEM_INSTRUCTION fallback — used when content_config singleton doesn't exist yet in Sanity.
// The live value is fetched from getConfigValue() inside stepEnriching().
const SYSTEM_INSTRUCTION_FALLBACK = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson.

Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles:
- Start with a BOLD claim or surprising fact that makes people stop scrolling
Expand Down Expand Up @@ -813,7 +826,8 @@ export async function GET(request: NextRequest) {
const results: StepResult[] = [];

// Phase 1: Stuck detection — runs FIRST, no external API calls
const stuckResults = await flagStuckDocs(docs, sanity);
const stuckThresholds = await buildStuckThresholds();
const stuckResults = await flagStuckDocs(docs, sanity, stuckThresholds);
results.push(...stuckResults);

// Remove flagged docs from further processing
Expand Down
29 changes: 25 additions & 4 deletions app/api/cron/ingest/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type { NextRequest } from "next/server";

import { generateWithGemini, stripCodeFences } from "@/lib/gemini";
import { writeClient } from "@/lib/sanity-write-client";
import { getConfigValue } from "@/lib/config";
import { discoverTrends, type TrendResult } from "@/lib/services/trend-discovery";
import type { ResearchPayload } from "@/lib/services/research";
import { NotebookLMClient } from "@/lib/services/notebooklm/client";
Expand Down Expand Up @@ -111,7 +112,9 @@ const FALLBACK_TRENDS: TrendResult[] = [
// Gemini Script Generation
// ---------------------------------------------------------------------------

const SYSTEM_INSTRUCTION = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson.
// SYSTEM_INSTRUCTION fallback — used when content_config singleton doesn't exist yet in Sanity.
// The live value is fetched from getConfigValue() inside the GET handler.
const SYSTEM_INSTRUCTION_FALLBACK = `You are a content strategist and scriptwriter for CodingCat.dev, a web development education channel run by Alex Patterson.

Your style is inspired by Cleo Abram's "Huge If True" — you make complex technical topics feel exciting, accessible, and important. Key principles:
- Start with a BOLD claim or surprising fact that makes people stop scrolling
Expand Down Expand Up @@ -330,10 +333,11 @@ async function createSanityDocuments(
script: GeneratedScript,
criticResult: CriticResult,
trends: TrendResult[],
qualityThreshold: number,
research?: ResearchPayload,
researchMeta?: { notebookId: string; taskId: string },
) {
const isFlagged = criticResult.score < 50;
const isFlagged = criticResult.score < qualityThreshold;
// When research is in-flight, status is "researching" (check-research cron will transition to script_ready)
const isResearching = !!researchMeta?.notebookId;
const status = isFlagged ? "flagged" : isResearching ? "researching" : "script_ready";
Expand Down Expand Up @@ -404,6 +408,23 @@ export async function GET(request: NextRequest) {
}

try {
// Fetch config values once per invocation (5-min in-memory cache)
const SYSTEM_INSTRUCTION = await getConfigValue(
"content_config",
"systemInstruction",
SYSTEM_INSTRUCTION_FALLBACK,
);
const enableNotebookLmResearch = await getConfigValue(
"pipeline_config",
"enableNotebookLmResearch",
false,
);
const qualityThreshold = await getConfigValue(
"pipeline_config",
"qualityThreshold",
50,
);

// Step 1: Discover trending topics (replaces fetchTrendingTopics)
console.log("[CRON/ingest] Discovering trending topics...");
let trends: TrendResult[];
Expand All @@ -425,7 +446,7 @@ export async function GET(request: NextRequest) {
// When research is enabled, we create a notebook and start research
// but DON'T wait for it — the check-research cron will poll and enrich later
let researchMeta: { notebookId: string; taskId: string } | undefined;
if (process.env.ENABLE_NOTEBOOKLM_RESEARCH === "true") {
if (enableNotebookLmResearch) {
console.log(`[CRON/ingest] Starting fire-and-forget research on: "${trends[0].topic}"...`);
try {
const auth = await initAuth();
Expand Down Expand Up @@ -494,7 +515,7 @@ export async function GET(request: NextRequest) {
);

console.log("[CRON/ingest] Creating Sanity documents...");
const result = await createSanityDocuments(script, criticResult, trends, undefined, researchMeta);
const result = await createSanityDocuments(script, criticResult, trends, qualityThreshold, undefined, researchMeta);

console.log("[CRON/ingest] Done!", result);

Expand Down
Loading