piotr/innercontext
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: AI photo analysis for skin snapshots

Browse source 
Add POST /skincare/analyze-photos endpoint that accepts 1–3 skin
photos, sends them to Gemini vision, and returns a structured
SkinPhotoAnalysisResponse for pre-filling the snapshot form.

Extract shared Gemini client setup into innercontext/llm.py
(get_gemini_client) so both products and skincare use a single
default model (gemini-flash-latest) and API key check.

Frontend: AI photo card on /skin page with file picker, previews,
and auto-fill of all form fields from the analysis result.
New fields (skin_type, sebum_tzone, sebum_cheeks) added to form
and server action.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Piotr Oleszczyk 2026-02-28 12:47:51 +01:00
parent cc25ac4e65
commit 66ee473deb
8 changed files with 356 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

26
frontend/src/lib/api.ts
View file

@ -218,3 +218,29 @@ export const updateSkinSnapshot = (
body: Record<string, unknown>
): Promise<SkinConditionSnapshot> => api.patch(`/skincare/${id}`, body);
export const deleteSkinSnapshot = (id: string): Promise<void> => api.del(`/skincare/${id}`);
export interface SkinPhotoAnalysisResponse {
overall_state?: string;
trend?: string;
skin_type?: string;
hydration_level?: number;
sebum_tzone?: number;
sebum_cheeks?: number;
sensitivity_level?: number;
barrier_state?: string;
active_concerns?: string[];
risks?: string[];
priorities?: string[];
notes?: string;
}
export async function analyzeSkinPhotos(files: File[]): Promise<SkinPhotoAnalysisResponse> {
const body = new FormData();
for (const file of files) body.append('photos', file);
const res = await fetch(`${PUBLIC_API_BASE}/skincare/analyze-photos`, { method: 'POST', body });
if (!res.ok) {
const detail = await res.json().catch(() => ({ detail: res.statusText }));
throw new Error(detail?.detail ?? res.statusText);
}
return res.json();
}

7
frontend/src/routes/skin/+page.server.ts
View file

@ -29,6 +29,10 @@ export const actions: Actions = {
.map((c) => c.trim())
.filter(Boolean) ?? [];
const skin_type = form.get('skin_type') as string;
const sebum_tzone = form.get('sebum_tzone') as string;
const sebum_cheeks = form.get('sebum_cheeks') as string;
const body: Record<string, unknown> = { snapshot_date, active_concerns };
if (overall_state) body.overall_state = overall_state;
if (trend) body.trend = trend;
@ -36,6 +40,9 @@ export const actions: Actions = {
if (hydration_level) body.hydration_level = Number(hydration_level);
if (sensitivity_level) body.sensitivity_level = Number(sensitivity_level);
if (barrier_state) body.barrier_state = barrier_state;
if (skin_type) body.skin_type = skin_type;
if (sebum_tzone) body.sebum_tzone = Number(sebum_tzone);
if (sebum_cheeks) body.sebum_cheeks = Number(sebum_cheeks);
try {
await createSkinSnapshot(body);

189
frontend/src/routes/skin/+page.svelte
View file

@ -1,6 +1,7 @@
<script lang="ts">
import { enhance } from '$app/forms';
import type { ActionData, PageData } from './$types';
import { analyzeSkinPhotos } from '$lib/api';
import { Badge } from '$lib/components/ui/badge';
import { Button } from '$lib/components/ui/button';
import { Card, CardContent, CardHeader, CardTitle } from '$lib/components/ui/card';
@ -13,6 +14,7 @@
const states = ['excellent', 'good', 'fair', 'poor'];
const trends = ['improving', 'stable', 'worsening', 'fluctuating'];
const barrierStates = ['intact', 'mildly_compromised', 'compromised'];
const skinTypes = ['dry', 'oily', 'combination', 'sensitive', 'normal', 'acne_prone'];
const stateColors: Record<string, string> = {
excellent: 'bg-green-100 text-green-800',
@ -22,13 +24,62 @@
};
let showForm = $state(false);
// Form state (bound to inputs so AI can pre-fill)
let snapshotDate = $state(new Date().toISOString().slice(0, 10));
let overallState = $state('');
let trend = $state('');
let barrierState = $state('');
let skinType = $state('');
let hydrationLevel = $state('');
let sensitivityLevel = $state('');
let sebumTzone = $state('');
let sebumCheeks = $state('');
let activeConcernsRaw = $state('');
let notes = $state('');
// AI photo analysis state
let aiPanelOpen = $state(false);
let selectedFiles = $state<File[]>([]);
let previewUrls = $state<string[]>([]);
let aiLoading = $state(false);
let aiError = $state('');
const sortedSnapshots = $derived(
[...data.snapshots].sort((a, b) => b.snapshot_date.localeCompare(a.snapshot_date))
);
function handleFileSelect(e: Event) {
const input = e.target as HTMLInputElement;
const files = Array.from(input.files ?? []).slice(0, 3);
selectedFiles = files;
previewUrls.forEach(URL.revokeObjectURL);
previewUrls = files.map((f) => URL.createObjectURL(f));
}
async function analyzePhotos() {
if (!selectedFiles.length) return;
aiLoading = true;
aiError = '';
try {
const r = await analyzeSkinPhotos(selectedFiles);
if (r.overall_state) overallState = r.overall_state;
if (r.trend) trend = r.trend;
if (r.skin_type) skinType = r.skin_type;
if (r.barrier_state) barrierState = r.barrier_state;
if (r.hydration_level != null) hydrationLevel = String(r.hydration_level);
if (r.sensitivity_level != null) sensitivityLevel = String(r.sensitivity_level);
if (r.sebum_tzone != null) sebumTzone = String(r.sebum_tzone);
if (r.sebum_cheeks != null) sebumCheeks = String(r.sebum_cheeks);
if (r.active_concerns?.length) activeConcernsRaw = r.active_concerns.join(', ');
if (r.notes) notes = r.notes;
aiPanelOpen = false;
} catch (e) {
aiError = (e as Error).message;
} finally {
aiLoading = false;
}
}
</script>
<svelte:head><title>Skin — innercontext</title></svelte:head>
@ -52,14 +103,67 @@
{/if}
{#if showForm}
<!-- AI photo analysis card -->
<Card>
<CardHeader>
<button
type="button"
class="flex w-full items-center justify-between text-left"
onclick={() => (aiPanelOpen = !aiPanelOpen)}
>
<CardTitle>AI analysis from photos</CardTitle>
<span class="text-sm text-muted-foreground">{aiPanelOpen ? '▲' : '▼'}</span>
</button>
</CardHeader>
{#if aiPanelOpen}
<CardContent class="space-y-3">
<p class="text-sm text-muted-foreground">
Upload 1–3 photos of your skin. AI will pre-fill the form fields below.
</p>
<input
type="file"
accept="image/jpeg,image/png,image/webp"
multiple
onchange={handleFileSelect}
class="block w-full text-sm text-muted-foreground
file:mr-4 file:rounded-md file:border-0 file:bg-primary
file:px-3 file:py-1.5 file:text-sm file:font-medium file:text-primary-foreground"
/>
{#if previewUrls.length}
<div class="flex flex-wrap gap-2">
{#each previewUrls as url (url)}
<img src={url} alt="skin preview" class="h-24 w-24 rounded-md object-cover border" />
{/each}
</div>
{/if}
{#if aiError}
<p class="text-sm text-destructive">{aiError}</p>
{/if}
<Button
type="button"
onclick={analyzePhotos}
disabled={aiLoading || !selectedFiles.length}
>
{aiLoading ? 'Analyzing…' : 'Analyze photos'}
</Button>
</CardContent>
{/if}
</Card>
<!-- New snapshot form -->
<Card>
<CardHeader><CardTitle>New skin snapshot</CardTitle></CardHeader>
<CardContent>
<form method="POST" action="?/create" use:enhance class="grid grid-cols-2 gap-4">
<div class="space-y-1">
<Label for="snapshot_date">Date *</Label>
<Input id="snapshot_date" name="snapshot_date" type="date"
value={new Date().toISOString().slice(0, 10)} required />
<Input
id="snapshot_date"
name="snapshot_date"
type="date"
bind:value={snapshotDate}
required
/>
</div>
<div class="space-y-1">
<Label>Overall state</Label>
@ -67,7 +171,7 @@
<Select type="single" value={overallState} onValueChange={(v) => (overallState = v)}>
<SelectTrigger>{overallState || 'Select'}</SelectTrigger>
<SelectContent>
{#each states as s}
{#each states as s (s)}
<SelectItem value={s}>{s}</SelectItem>
{/each}
</SelectContent>
@ -79,19 +183,33 @@
<Select type="single" value={trend} onValueChange={(v) => (trend = v)}>
<SelectTrigger>{trend || 'Select'}</SelectTrigger>
<SelectContent>
{#each trends as t}
{#each trends as t (t)}
<SelectItem value={t}>{t}</SelectItem>
{/each}
</SelectContent>
</Select>
</div>
<div class="space-y-1">
<Label>Skin type</Label>
<input type="hidden" name="skin_type" value={skinType} />
<Select type="single" value={skinType} onValueChange={(v) => (skinType = v)}>
<SelectTrigger>{skinType ? skinType.replace(/_/g, ' ') : 'Select'}</SelectTrigger>
<SelectContent>
{#each skinTypes as st (st)}
<SelectItem value={st}>{st.replace(/_/g, ' ')}</SelectItem>
{/each}
</SelectContent>
</Select>
</div>
<div class="space-y-1">
<Label>Barrier state</Label>
<input type="hidden" name="barrier_state" value={barrierState} />
<Select type="single" value={barrierState} onValueChange={(v) => (barrierState = v)}>
<SelectTrigger>{barrierState ? barrierState.replace(/_/g, ' ') : 'Select'}</SelectTrigger>
<SelectTrigger
>{barrierState ? barrierState.replace(/_/g, ' ') : 'Select'}</SelectTrigger
>
<SelectContent>
{#each barrierStates as b}
{#each barrierStates as b (b)}
<SelectItem value={b}>{b.replace(/_/g, ' ')}</SelectItem>
{/each}
</SelectContent>
@ -99,19 +217,60 @@
</div>
<div class="space-y-1">
<Label for="hydration_level">Hydration (1–5)</Label>
<Input id="hydration_level" name="hydration_level" type="number" min="1" max="5" />
<Input
id="hydration_level"
name="hydration_level"
type="number"
min="1"
max="5"
bind:value={hydrationLevel}
/>
</div>
<div class="space-y-1">
<Label for="sensitivity_level">Sensitivity (1–5)</Label>
<Input id="sensitivity_level" name="sensitivity_level" type="number" min="1" max="5" />
<Input
id="sensitivity_level"
name="sensitivity_level"
type="number"
min="1"
max="5"
bind:value={sensitivityLevel}
/>
</div>
<div class="space-y-1">
<Label for="sebum_tzone">Sebum T-zone (1–5)</Label>
<Input
id="sebum_tzone"
name="sebum_tzone"
type="number"
min="1"
max="5"
bind:value={sebumTzone}
/>
</div>
<div class="space-y-1">
<Label for="sebum_cheeks">Sebum cheeks (1–5)</Label>
<Input
id="sebum_cheeks"
name="sebum_cheeks"
type="number"
min="1"
max="5"
bind:value={sebumCheeks}
/>
</div>
<div class="space-y-1 col-span-2">
<Label for="active_concerns">Active concerns (comma-separated)</Label>
<Input id="active_concerns" name="active_concerns" placeholder="acne, redness, dehydration" />
<Input
id="active_concerns"
name="active_concerns"
placeholder="acne, redness, dehydration"
bind:value={activeConcernsRaw}
/>
</div>
<div class="space-y-1 col-span-2">
<Label for="notes">Notes</Label>
<Input id="notes" name="notes" />
<Input id="notes" name="notes" bind:value={notes} />
</div>
<div class="col-span-2">
<Button type="submit">Add snapshot</Button>
@ -122,14 +281,18 @@
{/if}
<div class="space-y-4">
{#each sortedSnapshots as snap}
{#each sortedSnapshots as snap (snap.id)}
<Card>
<CardContent class="pt-4">
<div class="flex items-center justify-between mb-3">
<span class="font-medium">{snap.snapshot_date}</span>
<div class="flex gap-2">
{#if snap.overall_state}
<span class="rounded-full px-2 py-0.5 text-xs font-medium {stateColors[snap.overall_state] ?? ''}">
<span
class="rounded-full px-2 py-0.5 text-xs font-medium {stateColors[
snap.overall_state
] ?? ''}"
>
{snap.overall_state}
</span>
{/if}
@ -160,7 +323,7 @@
</div>
{#if snap.active_concerns.length}
<div class="flex flex-wrap gap-1">
{#each snap.active_concerns as c}
{#each snap.active_concerns as c (c)}
<Badge variant="secondary" class="text-xs">{c.replace(/_/g, ' ')}</Badge>
{/each}
</div>