From c2eecc2f7c0855b3a530ebc7a2ea763b6c4e913b Mon Sep 17 00:00:00 2001 From: Ender Date: Sat, 25 Oct 2025 17:52:39 +0200 Subject: [PATCH] feat: add reference image support for AI content generation --- apps/admin/src/components/EditorShell.tsx | 6 ++ .../admin/src/components/steps/StepAssets.tsx | 13 +++- .../src/components/steps/StepGenerate.tsx | 25 ++++++- apps/admin/src/hooks/usePostEditor.ts | 16 ++++- apps/admin/src/services/ai.ts | 1 + apps/admin/src/services/posts.ts | 1 + apps/api/src/ai-generate.ts | 67 +++++++++++++++++-- 7 files changed, 120 insertions(+), 9 deletions(-) diff --git a/apps/admin/src/components/EditorShell.tsx b/apps/admin/src/components/EditorShell.tsx index a7f480d..f702c2d 100644 --- a/apps/admin/src/components/EditorShell.tsx +++ b/apps/admin/src/components/EditorShell.tsx @@ -29,6 +29,7 @@ export default function EditorShell({ onLogout: _onLogout, initialPostId, onBack previewLoading, previewError, genImageKeys, + referenceImageKeys, generatedDraft, imagePlaceholders, generationSources, @@ -48,6 +49,7 @@ export default function EditorShell({ onLogout: _onLogout, initialPostId, onBack publishToGhost, refreshPreview, toggleGenImage, + toggleReferenceImage, triggerAutoSave, triggerImmediateAutoSave, } = usePostEditor(initialPostId); @@ -132,6 +134,8 @@ export default function EditorShell({ onLogout: _onLogout, initialPostId, onBack onSetFeature={(url: string) => setMeta(m => ({ ...m, featureImage: url }))} selectedKeys={genImageKeys} onToggleSelect={toggleGenImage} + referenceKeys={referenceImageKeys} + onToggleReference={toggleReferenceImage} onAudioRemoved={triggerImmediateAutoSave} /> @@ -148,7 +152,9 @@ export default function EditorShell({ onLogout: _onLogout, initialPostId, onBack void; selectedKeys?: string[]; onToggleSelect?: (key: string) => void; + referenceKeys?: string[]; + onToggleReference?: (key: string) => void; onAudioRemoved?: () => void; }) { return ( @@ -40,7 +44,7 @@ export default function StepAssets({ onAudioRemoved={onAudioRemoved} /> - + + + + ); diff --git a/apps/admin/src/components/steps/StepGenerate.tsx b/apps/admin/src/components/steps/StepGenerate.tsx index 5a9ea3f..89d2e54 100644 --- a/apps/admin/src/components/steps/StepGenerate.tsx +++ b/apps/admin/src/components/steps/StepGenerate.tsx @@ -9,7 +9,9 @@ import type { Clip } from './StepAssets'; export default function StepGenerate({ postClips, genImageKeys, + referenceImageKeys, onToggleGenImage, + onToggleReference, promptText, onChangePrompt, generatedDraft, @@ -21,7 +23,9 @@ export default function StepGenerate({ }: { postClips: Clip[]; genImageKeys: string[]; + referenceImageKeys: string[]; onToggleGenImage: (key: string) => void; + onToggleReference: (key: string) => void; promptText: string; onChangePrompt: (v: string) => void; generatedDraft: string; @@ -59,9 +63,22 @@ export default function StepGenerate({ - {/* Selected images */} - - + {/* Content images */} + + {genImageKeys.length > 0 ? ( + + ) : ( + (No content images selected) + )} + + + {/* Reference images */} + + {referenceImageKeys.length > 0 ? ( + + ) : ( + (No reference images selected) + )} {/* Prompt */} @@ -114,11 +131,13 @@ export default function StepGenerate({ .map(c => c.transcript!); const imageUrls = genImageKeys.map(key => `/api/media/obj?key=${encodeURIComponent(key)}`); + const referenceUrls = referenceImageKeys.map(key => `/api/media/obj?key=${encodeURIComponent(key)}`); const result = await generateDraft({ prompt: promptText, audioTranscriptions: transcriptions.length > 0 ? transcriptions : undefined, selectedImageUrls: imageUrls.length > 0 ? imageUrls : undefined, + referenceImageUrls: referenceUrls.length > 0 ? referenceUrls : undefined, useWebSearch, }); diff --git a/apps/admin/src/hooks/usePostEditor.ts b/apps/admin/src/hooks/usePostEditor.ts index b656e9e..031180d 100644 --- a/apps/admin/src/hooks/usePostEditor.ts +++ b/apps/admin/src/hooks/usePostEditor.ts @@ -16,6 +16,7 @@ export function usePostEditor(initialPostId?: string | null) { const [previewLoading, setPreviewLoading] = useState(false); const [previewError, setPreviewError] = useState(null); const [genImageKeys, setGenImageKeys] = useState([]); + const [referenceImageKeys, setReferenceImageKeys] = useState([]); const [generatedDraft, setGeneratedDraft] = useState(''); const [imagePlaceholders, setImagePlaceholders] = useState([]); const [generationSources, setGenerationSources] = useState>([]); @@ -44,6 +45,7 @@ export function usePostEditor(initialPostId?: string | null) { if (data.status) setPostStatus(data.status); setPromptText(data.prompt || ''); if (Array.isArray(data.selectedImageKeys)) setGenImageKeys(data.selectedImageKeys); + if (Array.isArray(data.referenceImageKeys)) setReferenceImageKeys(data.referenceImageKeys); if (data.generatedDraft) setGeneratedDraft(data.generatedDraft); if (Array.isArray(data.imagePlaceholders)) setImagePlaceholders(data.imagePlaceholders); if (Array.isArray(data.generationSources)) setGenerationSources(data.generationSources); @@ -64,6 +66,7 @@ export function usePostEditor(initialPostId?: string | null) { status: postStatus, prompt: promptText || undefined, selectedImageKeys: genImageKeys.length > 0 ? genImageKeys : undefined, + referenceImageKeys: referenceImageKeys.length > 0 ? referenceImageKeys : undefined, generatedDraft: generatedDraft || undefined, imagePlaceholders: imagePlaceholders.length > 0 ? imagePlaceholders : undefined, generationSources: generationSources.length > 0 ? generationSources : undefined, @@ -78,7 +81,7 @@ export function usePostEditor(initialPostId?: string | null) { setToast({ open: true, message: 'Post saved', severity: 'success' }); } return data; - }, [postId, draft, meta, postStatus, promptText, genImageKeys, generatedDraft, imagePlaceholders, generationSources]); + }, [postId, draft, meta, postStatus, promptText, genImageKeys, referenceImageKeys, generatedDraft, imagePlaceholders, generationSources]); const deletePost = async () => { if (!postId) return; @@ -154,6 +157,15 @@ export function usePostEditor(initialPostId?: string | null) { }); }, [triggerImmediateAutoSave]); + const toggleReferenceImage = useCallback((key: string) => { + setReferenceImageKeys(prev => { + const newKeys = prev.includes(key) ? prev.filter(k => k !== key) : [...prev, key]; + // Trigger immediate auto-save after state update + setTimeout(() => triggerImmediateAutoSave(), 0); + return newKeys; + }); + }, [triggerImmediateAutoSave]); + // Cleanup timeout on unmount useEffect(() => { return () => { @@ -177,6 +189,7 @@ export function usePostEditor(initialPostId?: string | null) { previewLoading, previewError, genImageKeys, + referenceImageKeys, generatedDraft, imagePlaceholders, generationSources, @@ -197,6 +210,7 @@ export function usePostEditor(initialPostId?: string | null) { publishToGhost, refreshPreview, toggleGenImage, + toggleReferenceImage, triggerAutoSave, triggerImmediateAutoSave, } as const; diff --git a/apps/admin/src/services/ai.ts b/apps/admin/src/services/ai.ts index 36f9a1d..5b5aedc 100644 --- a/apps/admin/src/services/ai.ts +++ b/apps/admin/src/services/ai.ts @@ -2,6 +2,7 @@ export async function generateDraft(payload: { prompt: string; audioTranscriptions?: string[]; selectedImageUrls?: string[]; + referenceImageUrls?: string[]; useWebSearch?: boolean; }) { const res = await fetch('/api/ai/generate', { diff --git a/apps/admin/src/services/posts.ts b/apps/admin/src/services/posts.ts index f2bc17c..53c0daa 100644 --- a/apps/admin/src/services/posts.ts +++ b/apps/admin/src/services/posts.ts @@ -8,6 +8,7 @@ export type SavePostPayload = { status?: 'inbox' | 'editing' | 'ready_for_publish' | 'published' | 'archived'; prompt?: string; selectedImageKeys?: string[]; + referenceImageKeys?: string[]; generatedDraft?: string; imagePlaceholders?: string[]; generationSources?: Array<{ title: string; url: string }>; diff --git a/apps/api/src/ai-generate.ts b/apps/api/src/ai-generate.ts index 69d4e2f..2b51f87 100644 --- a/apps/api/src/ai-generate.ts +++ b/apps/api/src/ai-generate.ts @@ -3,6 +3,7 @@ import OpenAI from 'openai'; import { db } from './db'; import { settings } from './db/schema'; import { eq } from 'drizzle-orm'; +import { getPresignedUrl } from './storage/s3'; const router = express.Router(); @@ -33,11 +34,13 @@ router.post('/generate', async (req, res) => { prompt, audioTranscriptions, selectedImageUrls, + referenceImageUrls, useWebSearch = false, } = req.body as { prompt: string; audioTranscriptions?: string[]; selectedImageUrls?: string[]; + referenceImageUrls?: string[]; useWebSearch?: boolean; }; @@ -79,10 +82,50 @@ router.post('/generate', async (req, res) => { }); } - // Add information about available images + // Add information about available images (for article content) + // Note: We don't send these images to AI, just tell it how many are available if (selectedImageUrls && selectedImageUrls.length > 0) { - contextSection += '\n\nAVAILABLE IMAGES:\n'; - contextSection += `You have ${selectedImageUrls.length} images available. Use {{IMAGE:description}} placeholders where images should be inserted.\n`; + contextSection += '\n\nAVAILABLE IMAGES FOR ARTICLE:\n'; + contextSection += `You have ${selectedImageUrls.length} images available. Use {{IMAGE:description}} placeholders where images should be inserted in the article.\n`; + contextSection += `Important: You will NOT see these images. Just create descriptive placeholders based on where images would fit naturally in the content.\n`; + } + + // Generate presigned URLs for reference images (1 hour expiry) + const referenceImagePresignedUrls: string[] = []; + if (referenceImageUrls && referenceImageUrls.length > 0) { + console.log('[AI Generate] Creating presigned URLs for', referenceImageUrls.length, 'reference images'); + const bucket = process.env.S3_BUCKET || ''; + + for (const url of referenceImageUrls) { + try { + // Extract key from URL: /api/media/obj?key=images/abc.png + const keyMatch = url.match(/[?&]key=([^&]+)/); + if (keyMatch) { + const key = decodeURIComponent(keyMatch[1]); + const presignedUrl = await getPresignedUrl({ + bucket, + key, + expiresInSeconds: 3600 // 1 hour + }); + referenceImagePresignedUrls.push(presignedUrl); + console.log('[AI Generate] Generated presigned URL for:', key); + } + } catch (err) { + console.error('[AI Generate] Failed to create presigned URL:', err); + } + } + + // Add context about reference images + if (referenceImagePresignedUrls.length > 0) { + contextSection += '\n\nREFERENCE IMAGES (Context Only):\n'; + contextSection += `You will see ${referenceImagePresignedUrls.length} reference images below. These provide visual context to help you understand the topic better.\n`; + contextSection += `IMPORTANT: DO NOT create {{IMAGE:...}} placeholders for these reference images. They will NOT appear in the article.\n`; + contextSection += `Use these reference images to:\n`; + contextSection += `- Better understand the visual style and content\n`; + contextSection += `- Get inspiration for descriptions and explanations\n`; + contextSection += `- Understand technical details shown in screenshots\n`; + contextSection += `- Grasp the overall theme and aesthetic\n`; + } } const userPrompt = `${prompt}${contextSection}`; @@ -94,6 +137,22 @@ router.post('/generate', async (req, res) => { // Choose model based on web search requirement const model = useWebSearch ? 'gpt-4o-mini-search-preview-2025-03-11' : 'gpt-4o'; + // Build user message content with reference images (Vision API) + const userMessageContent: any[] = [ + { type: 'text', text: userPrompt } + ]; + + // Add reference images with presigned URLs (secure, 1-hour access) + if (referenceImagePresignedUrls.length > 0) { + console.log('[AI Generate] Adding', referenceImagePresignedUrls.length, 'reference images to vision'); + referenceImagePresignedUrls.forEach((url) => { + userMessageContent.push({ + type: 'image_url', + image_url: { url } + }); + }); + } + const completionParams: any = { model, messages: [ @@ -103,7 +162,7 @@ router.post('/generate', async (req, res) => { }, { role: 'user', - content: userPrompt, + content: userMessageContent, }, ], max_tokens: 4000,