feat: AI photo analysis for skin snapshots

Add POST /skincare/analyze-photos endpoint that accepts 1–3 skin
photos, sends them to Gemini vision, and returns a structured
SkinPhotoAnalysisResponse for pre-filling the snapshot form.

Extract shared Gemini client setup into innercontext/llm.py
(get_gemini_client) so both products and skincare use a single
default model (gemini-flash-latest) and API key check.

Frontend: AI photo card on /skin page with file picker, previews,
and auto-fill of all form fields from the analysis result.
New fields (skin_type, sebum_tzone, sebum_cheeks) added to form
and server action.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Piotr Oleszczyk 2026-02-28 12:47:51 +01:00
parent cc25ac4e65
commit 66ee473deb
8 changed files with 356 additions and 21 deletions

View file

@ -1,12 +1,16 @@
import json
from datetime import date
from typing import Optional
from uuid import UUID, uuid4
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from google.genai import types as genai_types
from pydantic import ValidationError
from sqlmodel import Session, SQLModel, select
from db import get_session
from innercontext.api.utils import get_or_404
from innercontext.llm import get_gemini_client
from innercontext.models import (
SkinConditionSnapshot,
SkinConditionSnapshotBase,
@ -51,11 +55,118 @@ class SnapshotUpdate(SQLModel):
notes: Optional[str] = None
class SkinPhotoAnalysisResponse(SQLModel):
overall_state: Optional[OverallSkinState] = None
trend: Optional[SkinTrend] = None
skin_type: Optional[SkinType] = None
hydration_level: Optional[int] = None
sebum_tzone: Optional[int] = None
sebum_cheeks: Optional[int] = None
sensitivity_level: Optional[int] = None
barrier_state: Optional[BarrierState] = None
active_concerns: Optional[list[SkinConcern]] = None
risks: Optional[list[str]] = None
priorities: Optional[list[str]] = None
notes: Optional[str] = None
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _skin_photo_system_prompt() -> str:
return """\
You are a dermatology-trained skin assessment AI. Analyze the provided photo(s) of a person's
skin and return a structured JSON assessment.
RULES:
- Return ONLY raw JSON no markdown fences, no explanation.
- Omit any field you cannot confidently determine from the photos. Do not guess.
- All enum values must exactly match the allowed strings listed below.
- Numeric metrics use a 15 scale (1 = minimal, 5 = maximal).
- For trends: only populate if you have strong visual cues; otherwise omit.
- risks and priorities: short English phrases, max 10 words each.
- notes: 24 sentence paragraph describing key observations.
ENUM VALUES:
overall_state: "excellent" | "good" | "fair" | "poor"
trend: "improving" | "stable" | "worsening" | "fluctuating"
skin_type: "dry" | "oily" | "combination" | "sensitive" | "normal" | "acne_prone"
barrier_state: "intact" | "mildly_compromised" | "compromised"
active_concerns: "acne" | "rosacea" | "hyperpigmentation" | "aging" | "dehydration" |
"redness" | "damaged_barrier" | "pore_visibility" | "uneven_texture" | "sebum_excess"
METRICS (int 15, omit if not assessable):
hydration_level: 1=very dehydrated/dull 5=plump/luminous
sebum_tzone: 1=very dry T-zone 5=very oily T-zone
sebum_cheeks: 1=very dry cheeks 5=very oily cheeks
sensitivity_level: 1=no visible signs 5=severe redness/reactivity
OUTPUT (all fields optional):
{"overall_state":, "trend":, "skin_type":, "hydration_level":,
"sebum_tzone":, "sebum_cheeks":, "sensitivity_level":,
"barrier_state":, "active_concerns":[], "risks":[], "priorities":[], "notes":}
"""
# ---------------------------------------------------------------------------
# Routes
# ---------------------------------------------------------------------------
MAX_IMAGE_BYTES = 5 * 1024 * 1024 # 5 MB
@router.post("/analyze-photos", response_model=SkinPhotoAnalysisResponse)
async def analyze_skin_photos(
photos: list[UploadFile] = File(...),
) -> SkinPhotoAnalysisResponse:
if not (1 <= len(photos) <= 3):
raise HTTPException(status_code=422, detail="Send between 1 and 3 photos.")
client, model = get_gemini_client()
allowed = {"image/jpeg", "image/png", "image/webp"}
parts: list[genai_types.Part] = []
for photo in photos:
if photo.content_type not in allowed:
raise HTTPException(status_code=422, detail=f"Unsupported type: {photo.content_type}")
data = await photo.read()
if len(data) > MAX_IMAGE_BYTES:
raise HTTPException(status_code=413, detail=f"{photo.filename} exceeds 5 MB.")
parts.append(genai_types.Part.from_bytes(data=data, mime_type=photo.content_type))
parts.append(
genai_types.Part.from_text(
"Analyze the skin condition visible in the above photo(s) and return the JSON assessment."
)
)
try:
response = client.models.generate_content(
model=model,
contents=parts,
config=genai_types.GenerateContentConfig(
system_instruction=_skin_photo_system_prompt(),
response_mime_type="application/json",
max_output_tokens=1024,
temperature=0.0,
),
)
except Exception as e:
raise HTTPException(status_code=502, detail=f"Gemini API error: {e}")
try:
parsed = json.loads(response.text)
except json.JSONDecodeError as e:
raise HTTPException(status_code=502, detail=f"LLM returned invalid JSON: {e}")
try:
return SkinPhotoAnalysisResponse.model_validate(parsed)
except ValidationError as e:
raise HTTPException(status_code=422, detail=e.errors())
@router.get("", response_model=list[SkinConditionSnapshotPublic])
def list_snapshots(
from_date: Optional[date] = None,