endery/voxblog
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: add reference image support for AI content generation

Browse Source
This commit is contained in:
Ender 2025-10-25 17:52:39 +02:00
parent 4fd46f4d24
commit c2eecc2f7c
7 changed files with 120 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/admin/src/components/EditorShell.tsx
View File

@ -29,6 +29,7 @@ export default function EditorShell({ onLogout: _onLogout, initialPostId, onBack
previewLoading,
previewError,
genImageKeys,
referenceImageKeys,
generatedDraft,
imagePlaceholders,
generationSources,
@ -48,6 +49,7 @@ export default function EditorShell({ onLogout: _onLogout, initialPostId, onBack
publishToGhost,
refreshPreview,
toggleGenImage,
toggleReferenceImage,
triggerAutoSave,
triggerImmediateAutoSave,
} = usePostEditor(initialPostId);
@ -132,6 +134,8 @@ export default function EditorShell({ onLogout: _onLogout, initialPostId, onBack
onSetFeature={(url: string) => setMeta(m => ({ ...m, featureImage: url }))}
selectedKeys={genImageKeys}
onToggleSelect={toggleGenImage}
referenceKeys={referenceImageKeys}
onToggleReference={toggleReferenceImage}
onAudioRemoved={triggerImmediateAutoSave}
/>
</StepContainer>
@ -148,7 +152,9 @@ export default function EditorShell({ onLogout: _onLogout, initialPostId, onBack
<StepGenerate
postClips={postClips}
genImageKeys={genImageKeys}
referenceImageKeys={referenceImageKeys}
onToggleGenImage={toggleGenImage}
onToggleReference={toggleReferenceImage}
promptText={promptText}
onChangePrompt={setPromptText}
generatedDraft={generatedDraft}

13
apps/admin/src/components/steps/StepAssets.tsx
View File

@ -14,6 +14,8 @@ export default function StepAssets({
onSetFeature,
selectedKeys,
onToggleSelect,
referenceKeys,
onToggleReference,
onAudioRemoved,
}: {
postId?: string | null;
@ -23,6 +25,8 @@ export default function StepAssets({
onSetFeature: (url: string) => void;
selectedKeys?: string[];
onToggleSelect?: (key: string) => void;
referenceKeys?: string[];
onToggleReference?: (key: string) => void;
onAudioRemoved?: () => void;
}) {
return (
@ -40,7 +44,7 @@ export default function StepAssets({
onAudioRemoved={onAudioRemoved}
/>
</CollapsibleSection>
<CollapsibleSection title="Media Library">
<CollapsibleSection title="Content Images (For Article)">
<MediaLibrary
onInsert={onInsertImage}
onSetFeature={onSetFeature}
@ -50,6 +54,13 @@ export default function StepAssets({
onToggleSelect={onToggleSelect}
/>
</CollapsibleSection>
<CollapsibleSection title="Reference Images (AI Context Only)">
<MediaLibrary
selectionMode
selectedKeys={referenceKeys}
onToggleSelect={onToggleReference}
/>
</CollapsibleSection>
</Stack>
</Box>
);

25
apps/admin/src/components/steps/StepGenerate.tsx
View File

@ -9,7 +9,9 @@ import type { Clip } from './StepAssets';
export default function StepGenerate({
postClips,
genImageKeys,
referenceImageKeys,
onToggleGenImage,
onToggleReference,
promptText,
onChangePrompt,
generatedDraft,
@ -21,7 +23,9 @@ export default function StepGenerate({
}: {
postClips: Clip[];
genImageKeys: string[];
referenceImageKeys: string[];
onToggleGenImage: (key: string) => void;
onToggleReference: (key: string) => void;
promptText: string;
onChangePrompt: (v: string) => void;
generatedDraft: string;
@ -59,9 +63,22 @@ export default function StepGenerate({
</Stack>
</CollapsibleSection>
{/* Selected images */}
<CollapsibleSection title="Selected Images">
<SelectedImages imageKeys={genImageKeys} onRemove={onToggleGenImage} />
{/* Content images */}
<CollapsibleSection title="Content Images (For Article)">
{genImageKeys.length > 0 ? (
<SelectedImages imageKeys={genImageKeys} onRemove={onToggleGenImage} />
) : (
<Typography variant="body2" sx={{ color: 'text.secondary' }}>(No content images selected)</Typography>
)}
</CollapsibleSection>
{/* Reference images */}
<CollapsibleSection title="Reference Images (AI Context Only)">
{referenceImageKeys.length > 0 ? (
<SelectedImages imageKeys={referenceImageKeys} onRemove={onToggleReference} />
) : (
<Typography variant="body2" sx={{ color: 'text.secondary' }}>(No reference images selected)</Typography>
)}
</CollapsibleSection>
{/* Prompt */}
@ -114,11 +131,13 @@ export default function StepGenerate({
.map(c => c.transcript!);
const imageUrls = genImageKeys.map(key => `/api/media/obj?key=${encodeURIComponent(key)}`);
const referenceUrls = referenceImageKeys.map(key => `/api/media/obj?key=${encodeURIComponent(key)}`);
const result = await generateDraft({
prompt: promptText,
audioTranscriptions: transcriptions.length > 0 ? transcriptions : undefined,
selectedImageUrls: imageUrls.length > 0 ? imageUrls : undefined,
referenceImageUrls: referenceUrls.length > 0 ? referenceUrls : undefined,
useWebSearch,
});

16
apps/admin/src/hooks/usePostEditor.ts
View File

@ -16,6 +16,7 @@ export function usePostEditor(initialPostId?: string | null) {
const [previewLoading, setPreviewLoading] = useState<boolean>(false);
const [previewError, setPreviewError] = useState<string | null>(null);
const [genImageKeys, setGenImageKeys] = useState<string[]>([]);
const [referenceImageKeys, setReferenceImageKeys] = useState<string[]>([]);
const [generatedDraft, setGeneratedDraft] = useState<string>('');
const [imagePlaceholders, setImagePlaceholders] = useState<string[]>([]);
const [generationSources, setGenerationSources] = useState<Array<{ title: string; url: string }>>([]);
@ -44,6 +45,7 @@ export function usePostEditor(initialPostId?: string | null) {
if (data.status) setPostStatus(data.status);
setPromptText(data.prompt || '');
if (Array.isArray(data.selectedImageKeys)) setGenImageKeys(data.selectedImageKeys);
if (Array.isArray(data.referenceImageKeys)) setReferenceImageKeys(data.referenceImageKeys);
if (data.generatedDraft) setGeneratedDraft(data.generatedDraft);
if (Array.isArray(data.imagePlaceholders)) setImagePlaceholders(data.imagePlaceholders);
if (Array.isArray(data.generationSources)) setGenerationSources(data.generationSources);
@ -64,6 +66,7 @@ export function usePostEditor(initialPostId?: string | null) {
status: postStatus,
prompt: promptText || undefined,
selectedImageKeys: genImageKeys.length > 0 ? genImageKeys : undefined,
referenceImageKeys: referenceImageKeys.length > 0 ? referenceImageKeys : undefined,
generatedDraft: generatedDraft || undefined,
imagePlaceholders: imagePlaceholders.length > 0 ? imagePlaceholders : undefined,
generationSources: generationSources.length > 0 ? generationSources : undefined,
@ -78,7 +81,7 @@ export function usePostEditor(initialPostId?: string | null) {
setToast({ open: true, message: 'Post saved', severity: 'success' });
}
return data;
}, [postId, draft, meta, postStatus, promptText, genImageKeys, generatedDraft, imagePlaceholders, generationSources]);
}, [postId, draft, meta, postStatus, promptText, genImageKeys, referenceImageKeys, generatedDraft, imagePlaceholders, generationSources]);
const deletePost = async () => {
if (!postId) return;
@ -154,6 +157,15 @@ export function usePostEditor(initialPostId?: string | null) {
});
}, [triggerImmediateAutoSave]);
const toggleReferenceImage = useCallback((key: string) => {
setReferenceImageKeys(prev => {
const newKeys = prev.includes(key) ? prev.filter(k => k !== key) : [...prev, key];
// Trigger immediate auto-save after state update
setTimeout(() => triggerImmediateAutoSave(), 0);
return newKeys;
});
}, [triggerImmediateAutoSave]);
// Cleanup timeout on unmount
useEffect(() => {
return () => {
@ -177,6 +189,7 @@ export function usePostEditor(initialPostId?: string | null) {
previewLoading,
previewError,
genImageKeys,
referenceImageKeys,
generatedDraft,
imagePlaceholders,
generationSources,
@ -197,6 +210,7 @@ export function usePostEditor(initialPostId?: string | null) {
publishToGhost,
refreshPreview,
toggleGenImage,
toggleReferenceImage,
triggerAutoSave,
triggerImmediateAutoSave,
} as const;

1
apps/admin/src/services/ai.ts
View File

@ -2,6 +2,7 @@ export async function generateDraft(payload: {
prompt: string;
audioTranscriptions?: string[];
selectedImageUrls?: string[];
referenceImageUrls?: string[];
useWebSearch?: boolean;
}) {
const res = await fetch('/api/ai/generate', {

1
apps/admin/src/services/posts.ts
View File

@ -8,6 +8,7 @@ export type SavePostPayload = {
status?: 'inbox' | 'editing' | 'ready_for_publish' | 'published' | 'archived';
prompt?: string;
selectedImageKeys?: string[];
referenceImageKeys?: string[];
generatedDraft?: string;
imagePlaceholders?: string[];
generationSources?: Array<{ title: string; url: string }>;

67
apps/api/src/ai-generate.ts
View File

@ -3,6 +3,7 @@ import OpenAI from 'openai';
import { db } from './db';
import { settings } from './db/schema';
import { eq } from 'drizzle-orm';
import { getPresignedUrl } from './storage/s3';
const router = express.Router();
@ -33,11 +34,13 @@ router.post('/generate', async (req, res) => {
prompt,
audioTranscriptions,
selectedImageUrls,
referenceImageUrls,
useWebSearch = false,
} = req.body as {
prompt: string;
audioTranscriptions?: string[];
selectedImageUrls?: string[];
referenceImageUrls?: string[];
useWebSearch?: boolean;
};
@ -79,10 +82,50 @@ router.post('/generate', async (req, res) => {
});
}
// Add information about available images
// Add information about available images (for article content)
// Note: We don't send these images to AI, just tell it how many are available
if (selectedImageUrls && selectedImageUrls.length > 0) {
contextSection += '\n\nAVAILABLE IMAGES:\n';
contextSection += `You have ${selectedImageUrls.length} images available. Use {{IMAGE:description}} placeholders where images should be inserted.\n`;
contextSection += '\n\nAVAILABLE IMAGES FOR ARTICLE:\n';
contextSection += `You have ${selectedImageUrls.length} images available. Use {{IMAGE:description}} placeholders where images should be inserted in the article.\n`;
contextSection += `Important: You will NOT see these images. Just create descriptive placeholders based on where images would fit naturally in the content.\n`;
}
// Generate presigned URLs for reference images (1 hour expiry)
const referenceImagePresignedUrls: string[] = [];
if (referenceImageUrls && referenceImageUrls.length > 0) {
console.log('[AI Generate] Creating presigned URLs for', referenceImageUrls.length, 'reference images');
const bucket = process.env.S3_BUCKET || '';
for (const url of referenceImageUrls) {
try {
// Extract key from URL: /api/media/obj?key=images/abc.png
const keyMatch = url.match(/[?&]key=([^&]+)/);
if (keyMatch) {
const key = decodeURIComponent(keyMatch[1]);
const presignedUrl = await getPresignedUrl({
bucket,
key,
expiresInSeconds: 3600 // 1 hour
});
referenceImagePresignedUrls.push(presignedUrl);
console.log('[AI Generate] Generated presigned URL for:', key);
}
} catch (err) {
console.error('[AI Generate] Failed to create presigned URL:', err);
}
}
// Add context about reference images
if (referenceImagePresignedUrls.length > 0) {
contextSection += '\n\nREFERENCE IMAGES (Context Only):\n';
contextSection += `You will see ${referenceImagePresignedUrls.length} reference images below. These provide visual context to help you understand the topic better.\n`;
contextSection += `IMPORTANT: DO NOT create {{IMAGE:...}} placeholders for these reference images. They will NOT appear in the article.\n`;
contextSection += `Use these reference images to:\n`;
contextSection += `- Better understand the visual style and content\n`;
contextSection += `- Get inspiration for descriptions and explanations\n`;
contextSection += `- Understand technical details shown in screenshots\n`;
contextSection += `- Grasp the overall theme and aesthetic\n`;
}
}
const userPrompt = `${prompt}${contextSection}`;
@ -94,6 +137,22 @@ router.post('/generate', async (req, res) => {
// Choose model based on web search requirement
const model = useWebSearch ? 'gpt-4o-mini-search-preview-2025-03-11' : 'gpt-4o';
// Build user message content with reference images (Vision API)
const userMessageContent: any[] = [
{ type: 'text', text: userPrompt }
];
// Add reference images with presigned URLs (secure, 1-hour access)
if (referenceImagePresignedUrls.length > 0) {
console.log('[AI Generate] Adding', referenceImagePresignedUrls.length, 'reference images to vision');
referenceImagePresignedUrls.forEach((url) => {
userMessageContent.push({
type: 'image_url',
image_url: { url }
});
});
}
const completionParams: any = {
model,
messages: [
@ -103,7 +162,7 @@ router.post('/generate', async (req, res) => {
},
{
role: 'user',
content: userPrompt,
content: userMessageContent,
},
],
max_tokens: 4000,