piotr/innercontext
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: AI photo analysis for skin snapshots

Browse source 
Add POST /skincare/analyze-photos endpoint that accepts 1–3 skin
photos, sends them to Gemini vision, and returns a structured
SkinPhotoAnalysisResponse for pre-filling the snapshot form.

Extract shared Gemini client setup into innercontext/llm.py
(get_gemini_client) so both products and skincare use a single
default model (gemini-flash-latest) and API key check.

Frontend: AI photo card on /skin page with file picker, previews,
and auto-fill of all form fields from the analysis result.
New fields (skin_type, sebum_tzone, sebum_cheeks) added to form
and server action.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Piotr Oleszczyk 2026-02-28 12:47:51 +01:00
parent cc25ac4e65
commit 66ee473deb
8 changed files with 356 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

9
backend/innercontext/api/products.py
View file

@ -1,17 +1,16 @@
import json
import os
from datetime import date
from typing import Optional
from uuid import UUID, uuid4
from fastapi import APIRouter, Depends, HTTPException, Query
from google import genai
from google.genai import types as genai_types
from pydantic import ValidationError
from sqlmodel import Session, SQLModel, select
from db import get_session
from innercontext.api.utils import get_or_404
from innercontext.llm import get_gemini_client
from innercontext.models import (
Product,
ProductBase,
@ -350,11 +349,7 @@ OUTPUT SCHEMA (all fields optional — omit what you cannot determine):
@router.post("/parse-text", response_model=ProductParseResponse)
def parse_product_text(data: ProductParseRequest) -> ProductParseResponse:
api_key = os.environ.get("GEMINI_API_KEY")
if not api_key:
raise HTTPException(status_code=503, detail="GEMINI_API_KEY not configured")
model = os.environ.get("GEMINI_MODEL", "gemini-flash-latest")
client = genai.Client(api_key=api_key)
client, model = get_gemini_client()
response = client.models.generate_content(
model=model,
contents=f"Extract product data from this text:\n\n{data.text}",

113
backend/innercontext/api/skincare.py
View file

@ -1,12 +1,16 @@
import json
from datetime import date
from typing import Optional
from uuid import UUID, uuid4
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from google.genai import types as genai_types
from pydantic import ValidationError
from sqlmodel import Session, SQLModel, select
from db import get_session
from innercontext.api.utils import get_or_404
from innercontext.llm import get_gemini_client
from innercontext.models import (
SkinConditionSnapshot,
SkinConditionSnapshotBase,
@ -51,11 +55,118 @@ class SnapshotUpdate(SQLModel):
notes: Optional[str] = None
class SkinPhotoAnalysisResponse(SQLModel):
overall_state: Optional[OverallSkinState] = None
trend: Optional[SkinTrend] = None
skin_type: Optional[SkinType] = None
hydration_level: Optional[int] = None
sebum_tzone: Optional[int] = None
sebum_cheeks: Optional[int] = None
sensitivity_level: Optional[int] = None
barrier_state: Optional[BarrierState] = None
active_concerns: Optional[list[SkinConcern]] = None
risks: Optional[list[str]] = None
priorities: Optional[list[str]] = None
notes: Optional[str] = None
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _skin_photo_system_prompt() -> str:
return """\
You are a dermatology-trained skin assessment AI. Analyze the provided photo(s) of a person's
skin and return a structured JSON assessment.
RULES:
- Return ONLY raw JSON — no markdown fences, no explanation.
- Omit any field you cannot confidently determine from the photos. Do not guess.
- All enum values must exactly match the allowed strings listed below.
- Numeric metrics use a 1–5 scale (1 = minimal, 5 = maximal).
- For trends: only populate if you have strong visual cues; otherwise omit.
- risks and priorities: short English phrases, max 10 words each.
- notes: 2–4 sentence paragraph describing key observations.
ENUM VALUES:
overall_state: "excellent" | "good" | "fair" | "poor"
trend: "improving" | "stable" | "worsening" | "fluctuating"
skin_type: "dry" | "oily" | "combination" | "sensitive" | "normal" | "acne_prone"
barrier_state: "intact" | "mildly_compromised" | "compromised"
active_concerns: "acne" | "rosacea" | "hyperpigmentation" | "aging" | "dehydration" |
"redness" | "damaged_barrier" | "pore_visibility" | "uneven_texture" | "sebum_excess"
METRICS (int 1–5, omit if not assessable):
hydration_level: 1=very dehydrated/dull → 5=plump/luminous
sebum_tzone: 1=very dry T-zone → 5=very oily T-zone
sebum_cheeks: 1=very dry cheeks → 5=very oily cheeks
sensitivity_level: 1=no visible signs → 5=severe redness/reactivity
OUTPUT (all fields optional):
{"overall_state":…, "trend":…, "skin_type":…, "hydration_level":…,
"sebum_tzone":…, "sebum_cheeks":…, "sensitivity_level":…,
"barrier_state":…, "active_concerns":[…], "risks":[…], "priorities":[…], "notes":…}
"""
# ---------------------------------------------------------------------------
# Routes
# ---------------------------------------------------------------------------
MAX_IMAGE_BYTES = 5 * 1024 * 1024 # 5 MB
@router.post("/analyze-photos", response_model=SkinPhotoAnalysisResponse)
async def analyze_skin_photos(
photos: list[UploadFile] = File(...),
) -> SkinPhotoAnalysisResponse:
if not (1 <= len(photos) <= 3):
raise HTTPException(status_code=422, detail="Send between 1 and 3 photos.")
client, model = get_gemini_client()
allowed = {"image/jpeg", "image/png", "image/webp"}
parts: list[genai_types.Part] = []
for photo in photos:
if photo.content_type not in allowed:
raise HTTPException(status_code=422, detail=f"Unsupported type: {photo.content_type}")
data = await photo.read()
if len(data) > MAX_IMAGE_BYTES:
raise HTTPException(status_code=413, detail=f"{photo.filename} exceeds 5 MB.")
parts.append(genai_types.Part.from_bytes(data=data, mime_type=photo.content_type))
parts.append(
genai_types.Part.from_text(
"Analyze the skin condition visible in the above photo(s) and return the JSON assessment."
)
)
try:
response = client.models.generate_content(
model=model,
contents=parts,
config=genai_types.GenerateContentConfig(
system_instruction=_skin_photo_system_prompt(),
response_mime_type="application/json",
max_output_tokens=1024,
temperature=0.0,
),
)
except Exception as e:
raise HTTPException(status_code=502, detail=f"Gemini API error: {e}")
try:
parsed = json.loads(response.text)
except json.JSONDecodeError as e:
raise HTTPException(status_code=502, detail=f"LLM returned invalid JSON: {e}")
try:
return SkinPhotoAnalysisResponse.model_validate(parsed)
except ValidationError as e:
raise HTTPException(status_code=422, detail=e.errors())
@router.get("", response_model=list[SkinConditionSnapshotPublic])
def list_snapshots(
from_date: Optional[date] = None,

21
backend/innercontext/llm.py Normal file
View file

@ -0,0 +1,21 @@
"""Shared helpers for Gemini API access."""
import os
from fastapi import HTTPException
from google import genai
_DEFAULT_MODEL = "gemini-flash-latest"
def get_gemini_client() -> tuple[genai.Client, str]:
"""Return an authenticated Gemini client and the configured model name.
Raises HTTP 503 if GEMINI_API_KEY is not set.
"""
api_key = os.environ.get("GEMINI_API_KEY")
if not api_key:
raise HTTPException(status_code=503, detail="GEMINI_API_KEY not configured")
model = os.environ.get("GEMINI_MODEL", _DEFAULT_MODEL)
return genai.Client(api_key=api_key), model

1
backend/pyproject.toml
View file

@ -9,6 +9,7 @@ dependencies = [
"google-genai>=1.65.0",
"psycopg>=3.3.3",
"python-dotenv>=1.2.1",
"python-multipart>=0.0.22",
"sqlmodel>=0.0.37",
"uvicorn[standard]>=0.34.0",
]

11
backend/uv.lock generated
View file

@ -459,6 +459,7 @@ dependencies = [
{ name = "google-genai" },
{ name = "psycopg" },
{ name = "python-dotenv" },
{ name = "python-multipart" },
{ name = "sqlmodel" },
{ name = "uvicorn", extra = ["standard"] },
]
@ -479,6 +480,7 @@ requires-dist = [
{ name = "google-genai", specifier = ">=1.65.0" },
{ name = "psycopg", specifier = ">=3.3.3" },
{ name = "python-dotenv", specifier = ">=1.2.1" },
{ name = "python-multipart", specifier = ">=0.0.22" },
{ name = "sqlmodel", specifier = ">=0.0.37" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" },
]
@ -710,6 +712,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" },
]
[[package]]
name = "python-multipart"
version = "0.0.22"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/94/01/979e98d542a70714b0cb2b6728ed0b7c46792b695e3eaec3e20711271ca3/python_multipart-0.0.22.tar.gz", hash = "sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58", size = 37612, upload-time = "2026-01-25T10:15:56.219Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1b/d0/397f9626e711ff749a95d96b7af99b9c566a9bb5129b8e4c10fc4d100304/python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155", size = 24579, upload-time = "2026-01-25T10:15:54.811Z" },
]
[[package]]
name = "pytokens"
version = "0.4.1"

26
frontend/src/lib/api.ts
View file

@ -218,3 +218,29 @@ export const updateSkinSnapshot = (
body: Record<string, unknown>
): Promise<SkinConditionSnapshot> => api.patch(`/skincare/${id}`, body);
export const deleteSkinSnapshot = (id: string): Promise<void> => api.del(`/skincare/${id}`);
export interface SkinPhotoAnalysisResponse {
overall_state?: string;
trend?: string;
skin_type?: string;
hydration_level?: number;
sebum_tzone?: number;
sebum_cheeks?: number;
sensitivity_level?: number;
barrier_state?: string;
active_concerns?: string[];
risks?: string[];
priorities?: string[];
notes?: string;
}
export async function analyzeSkinPhotos(files: File[]): Promise<SkinPhotoAnalysisResponse> {
const body = new FormData();
for (const file of files) body.append('photos', file);
const res = await fetch(`${PUBLIC_API_BASE}/skincare/analyze-photos`, { method: 'POST', body });
if (!res.ok) {
const detail = await res.json().catch(() => ({ detail: res.statusText }));
throw new Error(detail?.detail ?? res.statusText);
}
return res.json();
}

7
frontend/src/routes/skin/+page.server.ts
View file

@ -29,6 +29,10 @@ export const actions: Actions = {
.map((c) => c.trim())
.filter(Boolean) ?? [];
const skin_type = form.get('skin_type') as string;
const sebum_tzone = form.get('sebum_tzone') as string;
const sebum_cheeks = form.get('sebum_cheeks') as string;
const body: Record<string, unknown> = { snapshot_date, active_concerns };
if (overall_state) body.overall_state = overall_state;
if (trend) body.trend = trend;
@ -36,6 +40,9 @@ export const actions: Actions = {
if (hydration_level) body.hydration_level = Number(hydration_level);
if (sensitivity_level) body.sensitivity_level = Number(sensitivity_level);
if (barrier_state) body.barrier_state = barrier_state;
if (skin_type) body.skin_type = skin_type;
if (sebum_tzone) body.sebum_tzone = Number(sebum_tzone);
if (sebum_cheeks) body.sebum_cheeks = Number(sebum_cheeks);
try {
await createSkinSnapshot(body);

189
frontend/src/routes/skin/+page.svelte
View file

@ -1,6 +1,7 @@
<script lang="ts">
import { enhance } from '$app/forms';
import type { ActionData, PageData } from './$types';
import { analyzeSkinPhotos } from '$lib/api';
import { Badge } from '$lib/components/ui/badge';
import { Button } from '$lib/components/ui/button';
import { Card, CardContent, CardHeader, CardTitle } from '$lib/components/ui/card';
@ -13,6 +14,7 @@
const states = ['excellent', 'good', 'fair', 'poor'];
const trends = ['improving', 'stable', 'worsening', 'fluctuating'];
const barrierStates = ['intact', 'mildly_compromised', 'compromised'];
const skinTypes = ['dry', 'oily', 'combination', 'sensitive', 'normal', 'acne_prone'];
const stateColors: Record<string, string> = {
excellent: 'bg-green-100 text-green-800',
@ -22,13 +24,62 @@
};
let showForm = $state(false);
// Form state (bound to inputs so AI can pre-fill)
let snapshotDate = $state(new Date().toISOString().slice(0, 10));
let overallState = $state('');
let trend = $state('');
let barrierState = $state('');
let skinType = $state('');
let hydrationLevel = $state('');
let sensitivityLevel = $state('');
let sebumTzone = $state('');
let sebumCheeks = $state('');
let activeConcernsRaw = $state('');
let notes = $state('');
// AI photo analysis state
let aiPanelOpen = $state(false);
let selectedFiles = $state<File[]>([]);
let previewUrls = $state<string[]>([]);
let aiLoading = $state(false);
let aiError = $state('');
const sortedSnapshots = $derived(
[...data.snapshots].sort((a, b) => b.snapshot_date.localeCompare(a.snapshot_date))
);
function handleFileSelect(e: Event) {
const input = e.target as HTMLInputElement;
const files = Array.from(input.files ?? []).slice(0, 3);
selectedFiles = files;
previewUrls.forEach(URL.revokeObjectURL);
previewUrls = files.map((f) => URL.createObjectURL(f));
}
async function analyzePhotos() {
if (!selectedFiles.length) return;
aiLoading = true;
aiError = '';
try {
const r = await analyzeSkinPhotos(selectedFiles);
if (r.overall_state) overallState = r.overall_state;
if (r.trend) trend = r.trend;
if (r.skin_type) skinType = r.skin_type;
if (r.barrier_state) barrierState = r.barrier_state;
if (r.hydration_level != null) hydrationLevel = String(r.hydration_level);
if (r.sensitivity_level != null) sensitivityLevel = String(r.sensitivity_level);
if (r.sebum_tzone != null) sebumTzone = String(r.sebum_tzone);
if (r.sebum_cheeks != null) sebumCheeks = String(r.sebum_cheeks);
if (r.active_concerns?.length) activeConcernsRaw = r.active_concerns.join(', ');
if (r.notes) notes = r.notes;
aiPanelOpen = false;
} catch (e) {
aiError = (e as Error).message;
} finally {
aiLoading = false;
}
}
</script>
<svelte:head><title>Skin — innercontext</title></svelte:head>
@ -52,14 +103,67 @@
{/if}
{#if showForm}
<!-- AI photo analysis card -->
<Card>
<CardHeader>
<button
type="button"
class="flex w-full items-center justify-between text-left"
onclick={() => (aiPanelOpen = !aiPanelOpen)}
>
<CardTitle>AI analysis from photos</CardTitle>
<span class="text-sm text-muted-foreground">{aiPanelOpen ? '▲' : '▼'}</span>
</button>
</CardHeader>
{#if aiPanelOpen}
<CardContent class="space-y-3">
<p class="text-sm text-muted-foreground">
Upload 1–3 photos of your skin. AI will pre-fill the form fields below.
</p>
<input
type="file"
accept="image/jpeg,image/png,image/webp"
multiple
onchange={handleFileSelect}
class="block w-full text-sm text-muted-foreground
file:mr-4 file:rounded-md file:border-0 file:bg-primary
file:px-3 file:py-1.5 file:text-sm file:font-medium file:text-primary-foreground"
/>
{#if previewUrls.length}
<div class="flex flex-wrap gap-2">
{#each previewUrls as url (url)}
<img src={url} alt="skin preview" class="h-24 w-24 rounded-md object-cover border" />
{/each}
</div>
{/if}
{#if aiError}
<p class="text-sm text-destructive">{aiError}</p>
{/if}
<Button
type="button"
onclick={analyzePhotos}
disabled={aiLoading || !selectedFiles.length}
>
{aiLoading ? 'Analyzing…' : 'Analyze photos'}
</Button>
</CardContent>
{/if}
</Card>
<!-- New snapshot form -->
<Card>
<CardHeader><CardTitle>New skin snapshot</CardTitle></CardHeader>
<CardContent>
<form method="POST" action="?/create" use:enhance class="grid grid-cols-2 gap-4">
<div class="space-y-1">
<Label for="snapshot_date">Date *</Label>
<Input id="snapshot_date" name="snapshot_date" type="date"
value={new Date().toISOString().slice(0, 10)} required />
<Input
id="snapshot_date"
name="snapshot_date"
type="date"
bind:value={snapshotDate}
required
/>
</div>
<div class="space-y-1">
<Label>Overall state</Label>
@ -67,7 +171,7 @@
<Select type="single" value={overallState} onValueChange={(v) => (overallState = v)}>
<SelectTrigger>{overallState || 'Select'}</SelectTrigger>
<SelectContent>
{#each states as s}
{#each states as s (s)}
<SelectItem value={s}>{s}</SelectItem>
{/each}
</SelectContent>
@ -79,19 +183,33 @@
<Select type="single" value={trend} onValueChange={(v) => (trend = v)}>
<SelectTrigger>{trend || 'Select'}</SelectTrigger>
<SelectContent>
{#each trends as t}
{#each trends as t (t)}
<SelectItem value={t}>{t}</SelectItem>
{/each}
</SelectContent>
</Select>
</div>
<div class="space-y-1">
<Label>Skin type</Label>
<input type="hidden" name="skin_type" value={skinType} />
<Select type="single" value={skinType} onValueChange={(v) => (skinType = v)}>
<SelectTrigger>{skinType ? skinType.replace(/_/g, ' ') : 'Select'}</SelectTrigger>
<SelectContent>
{#each skinTypes as st (st)}
<SelectItem value={st}>{st.replace(/_/g, ' ')}</SelectItem>
{/each}
</SelectContent>
</Select>
</div>
<div class="space-y-1">
<Label>Barrier state</Label>
<input type="hidden" name="barrier_state" value={barrierState} />
<Select type="single" value={barrierState} onValueChange={(v) => (barrierState = v)}>
<SelectTrigger>{barrierState ? barrierState.replace(/_/g, ' ') : 'Select'}</SelectTrigger>
<SelectTrigger
>{barrierState ? barrierState.replace(/_/g, ' ') : 'Select'}</SelectTrigger
>
<SelectContent>
{#each barrierStates as b}
{#each barrierStates as b (b)}
<SelectItem value={b}>{b.replace(/_/g, ' ')}</SelectItem>
{/each}
</SelectContent>
@ -99,19 +217,60 @@
</div>
<div class="space-y-1">
<Label for="hydration_level">Hydration (1–5)</Label>
<Input id="hydration_level" name="hydration_level" type="number" min="1" max="5" />
<Input
id="hydration_level"
name="hydration_level"
type="number"
min="1"
max="5"
bind:value={hydrationLevel}
/>
</div>
<div class="space-y-1">
<Label for="sensitivity_level">Sensitivity (1–5)</Label>
<Input id="sensitivity_level" name="sensitivity_level" type="number" min="1" max="5" />
<Input
id="sensitivity_level"
name="sensitivity_level"
type="number"
min="1"
max="5"
bind:value={sensitivityLevel}
/>
</div>
<div class="space-y-1">
<Label for="sebum_tzone">Sebum T-zone (1–5)</Label>
<Input
id="sebum_tzone"
name="sebum_tzone"
type="number"
min="1"
max="5"
bind:value={sebumTzone}
/>
</div>
<div class="space-y-1">
<Label for="sebum_cheeks">Sebum cheeks (1–5)</Label>
<Input
id="sebum_cheeks"
name="sebum_cheeks"
type="number"
min="1"
max="5"
bind:value={sebumCheeks}
/>
</div>
<div class="space-y-1 col-span-2">
<Label for="active_concerns">Active concerns (comma-separated)</Label>
<Input id="active_concerns" name="active_concerns" placeholder="acne, redness, dehydration" />
<Input
id="active_concerns"
name="active_concerns"
placeholder="acne, redness, dehydration"
bind:value={activeConcernsRaw}
/>
</div>
<div class="space-y-1 col-span-2">
<Label for="notes">Notes</Label>
<Input id="notes" name="notes" />
<Input id="notes" name="notes" bind:value={notes} />
</div>
<div class="col-span-2">
<Button type="submit">Add snapshot</Button>
@ -122,14 +281,18 @@
{/if}
<div class="space-y-4">
{#each sortedSnapshots as snap}
{#each sortedSnapshots as snap (snap.id)}
<Card>
<CardContent class="pt-4">
<div class="flex items-center justify-between mb-3">
<span class="font-medium">{snap.snapshot_date}</span>
<div class="flex gap-2">
{#if snap.overall_state}
<span class="rounded-full px-2 py-0.5 text-xs font-medium {stateColors[snap.overall_state] ?? ''}">
<span
class="rounded-full px-2 py-0.5 text-xs font-medium {stateColors[
snap.overall_state
] ?? ''}"
>
{snap.overall_state}
</span>
{/if}
@ -160,7 +323,7 @@
</div>
{#if snap.active_concerns.length}
<div class="flex flex-wrap gap-1">
{#each snap.active_concerns as c}
{#each snap.active_concerns as c (c)}
<Badge variant="secondary" class="text-xs">{c.replace(/_/g, ' ')}</Badge>
{/each}
</div>