feat: AI photo analysis for skin snapshots

Add POST /skincare/analyze-photos endpoint that accepts 1–3 skin
photos, sends them to Gemini vision, and returns a structured
SkinPhotoAnalysisResponse for pre-filling the snapshot form.

Extract shared Gemini client setup into innercontext/llm.py
(get_gemini_client) so both products and skincare use a single
default model (gemini-flash-latest) and API key check.

Frontend: AI photo card on /skin page with file picker, previews,
and auto-fill of all form fields from the analysis result.
New fields (skin_type, sebum_tzone, sebum_cheeks) added to form
and server action.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Piotr Oleszczyk 2026-02-28 12:47:51 +01:00
parent cc25ac4e65
commit 66ee473deb
8 changed files with 356 additions and 21 deletions

View file

@ -1,17 +1,16 @@
import json
import os
from datetime import date
from typing import Optional
from uuid import UUID, uuid4
from fastapi import APIRouter, Depends, HTTPException, Query
from google import genai
from google.genai import types as genai_types
from pydantic import ValidationError
from sqlmodel import Session, SQLModel, select
from db import get_session
from innercontext.api.utils import get_or_404
from innercontext.llm import get_gemini_client
from innercontext.models import (
Product,
ProductBase,
@ -350,11 +349,7 @@ OUTPUT SCHEMA (all fields optional — omit what you cannot determine):
@router.post("/parse-text", response_model=ProductParseResponse)
def parse_product_text(data: ProductParseRequest) -> ProductParseResponse:
api_key = os.environ.get("GEMINI_API_KEY")
if not api_key:
raise HTTPException(status_code=503, detail="GEMINI_API_KEY not configured")
model = os.environ.get("GEMINI_MODEL", "gemini-flash-latest")
client = genai.Client(api_key=api_key)
client, model = get_gemini_client()
response = client.models.generate_content(
model=model,
contents=f"Extract product data from this text:\n\n{data.text}",

View file

@ -1,12 +1,16 @@
import json
from datetime import date
from typing import Optional
from uuid import UUID, uuid4
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from google.genai import types as genai_types
from pydantic import ValidationError
from sqlmodel import Session, SQLModel, select
from db import get_session
from innercontext.api.utils import get_or_404
from innercontext.llm import get_gemini_client
from innercontext.models import (
SkinConditionSnapshot,
SkinConditionSnapshotBase,
@ -51,11 +55,118 @@ class SnapshotUpdate(SQLModel):
notes: Optional[str] = None
class SkinPhotoAnalysisResponse(SQLModel):
overall_state: Optional[OverallSkinState] = None
trend: Optional[SkinTrend] = None
skin_type: Optional[SkinType] = None
hydration_level: Optional[int] = None
sebum_tzone: Optional[int] = None
sebum_cheeks: Optional[int] = None
sensitivity_level: Optional[int] = None
barrier_state: Optional[BarrierState] = None
active_concerns: Optional[list[SkinConcern]] = None
risks: Optional[list[str]] = None
priorities: Optional[list[str]] = None
notes: Optional[str] = None
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _skin_photo_system_prompt() -> str:
return """\
You are a dermatology-trained skin assessment AI. Analyze the provided photo(s) of a person's
skin and return a structured JSON assessment.
RULES:
- Return ONLY raw JSON no markdown fences, no explanation.
- Omit any field you cannot confidently determine from the photos. Do not guess.
- All enum values must exactly match the allowed strings listed below.
- Numeric metrics use a 15 scale (1 = minimal, 5 = maximal).
- For trends: only populate if you have strong visual cues; otherwise omit.
- risks and priorities: short English phrases, max 10 words each.
- notes: 24 sentence paragraph describing key observations.
ENUM VALUES:
overall_state: "excellent" | "good" | "fair" | "poor"
trend: "improving" | "stable" | "worsening" | "fluctuating"
skin_type: "dry" | "oily" | "combination" | "sensitive" | "normal" | "acne_prone"
barrier_state: "intact" | "mildly_compromised" | "compromised"
active_concerns: "acne" | "rosacea" | "hyperpigmentation" | "aging" | "dehydration" |
"redness" | "damaged_barrier" | "pore_visibility" | "uneven_texture" | "sebum_excess"
METRICS (int 15, omit if not assessable):
hydration_level: 1=very dehydrated/dull 5=plump/luminous
sebum_tzone: 1=very dry T-zone 5=very oily T-zone
sebum_cheeks: 1=very dry cheeks 5=very oily cheeks
sensitivity_level: 1=no visible signs 5=severe redness/reactivity
OUTPUT (all fields optional):
{"overall_state":, "trend":, "skin_type":, "hydration_level":,
"sebum_tzone":, "sebum_cheeks":, "sensitivity_level":,
"barrier_state":, "active_concerns":[], "risks":[], "priorities":[], "notes":}
"""
# ---------------------------------------------------------------------------
# Routes
# ---------------------------------------------------------------------------
MAX_IMAGE_BYTES = 5 * 1024 * 1024 # 5 MB
@router.post("/analyze-photos", response_model=SkinPhotoAnalysisResponse)
async def analyze_skin_photos(
photos: list[UploadFile] = File(...),
) -> SkinPhotoAnalysisResponse:
if not (1 <= len(photos) <= 3):
raise HTTPException(status_code=422, detail="Send between 1 and 3 photos.")
client, model = get_gemini_client()
allowed = {"image/jpeg", "image/png", "image/webp"}
parts: list[genai_types.Part] = []
for photo in photos:
if photo.content_type not in allowed:
raise HTTPException(status_code=422, detail=f"Unsupported type: {photo.content_type}")
data = await photo.read()
if len(data) > MAX_IMAGE_BYTES:
raise HTTPException(status_code=413, detail=f"{photo.filename} exceeds 5 MB.")
parts.append(genai_types.Part.from_bytes(data=data, mime_type=photo.content_type))
parts.append(
genai_types.Part.from_text(
"Analyze the skin condition visible in the above photo(s) and return the JSON assessment."
)
)
try:
response = client.models.generate_content(
model=model,
contents=parts,
config=genai_types.GenerateContentConfig(
system_instruction=_skin_photo_system_prompt(),
response_mime_type="application/json",
max_output_tokens=1024,
temperature=0.0,
),
)
except Exception as e:
raise HTTPException(status_code=502, detail=f"Gemini API error: {e}")
try:
parsed = json.loads(response.text)
except json.JSONDecodeError as e:
raise HTTPException(status_code=502, detail=f"LLM returned invalid JSON: {e}")
try:
return SkinPhotoAnalysisResponse.model_validate(parsed)
except ValidationError as e:
raise HTTPException(status_code=422, detail=e.errors())
@router.get("", response_model=list[SkinConditionSnapshotPublic])
def list_snapshots(
from_date: Optional[date] = None,

View file

@ -0,0 +1,21 @@
"""Shared helpers for Gemini API access."""
import os
from fastapi import HTTPException
from google import genai
_DEFAULT_MODEL = "gemini-flash-latest"
def get_gemini_client() -> tuple[genai.Client, str]:
"""Return an authenticated Gemini client and the configured model name.
Raises HTTP 503 if GEMINI_API_KEY is not set.
"""
api_key = os.environ.get("GEMINI_API_KEY")
if not api_key:
raise HTTPException(status_code=503, detail="GEMINI_API_KEY not configured")
model = os.environ.get("GEMINI_MODEL", _DEFAULT_MODEL)
return genai.Client(api_key=api_key), model

View file

@ -9,6 +9,7 @@ dependencies = [
"google-genai>=1.65.0",
"psycopg>=3.3.3",
"python-dotenv>=1.2.1",
"python-multipart>=0.0.22",
"sqlmodel>=0.0.37",
"uvicorn[standard]>=0.34.0",
]

11
backend/uv.lock generated
View file

@ -459,6 +459,7 @@ dependencies = [
{ name = "google-genai" },
{ name = "psycopg" },
{ name = "python-dotenv" },
{ name = "python-multipart" },
{ name = "sqlmodel" },
{ name = "uvicorn", extra = ["standard"] },
]
@ -479,6 +480,7 @@ requires-dist = [
{ name = "google-genai", specifier = ">=1.65.0" },
{ name = "psycopg", specifier = ">=3.3.3" },
{ name = "python-dotenv", specifier = ">=1.2.1" },
{ name = "python-multipart", specifier = ">=0.0.22" },
{ name = "sqlmodel", specifier = ">=0.0.37" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" },
]
@ -710,6 +712,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" },
]
[[package]]
name = "python-multipart"
version = "0.0.22"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/94/01/979e98d542a70714b0cb2b6728ed0b7c46792b695e3eaec3e20711271ca3/python_multipart-0.0.22.tar.gz", hash = "sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58", size = 37612, upload-time = "2026-01-25T10:15:56.219Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1b/d0/397f9626e711ff749a95d96b7af99b9c566a9bb5129b8e4c10fc4d100304/python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155", size = 24579, upload-time = "2026-01-25T10:15:54.811Z" },
]
[[package]]
name = "pytokens"
version = "0.4.1"