feat: AI photo analysis for skin snapshots
Add POST /skincare/analyze-photos endpoint that accepts 1–3 skin photos, sends them to Gemini vision, and returns a structured SkinPhotoAnalysisResponse for pre-filling the snapshot form. Extract shared Gemini client setup into innercontext/llm.py (get_gemini_client) so both products and skincare use a single default model (gemini-flash-latest) and API key check. Frontend: AI photo card on /skin page with file picker, previews, and auto-fill of all form fields from the analysis result. New fields (skin_type, sebum_tzone, sebum_cheeks) added to form and server action. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
cc25ac4e65
commit
66ee473deb
8 changed files with 356 additions and 21 deletions
|
|
@ -1,17 +1,16 @@
|
|||
import json
|
||||
import os
|
||||
from datetime import date
|
||||
from typing import Optional
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from google import genai
|
||||
from google.genai import types as genai_types
|
||||
from pydantic import ValidationError
|
||||
from sqlmodel import Session, SQLModel, select
|
||||
|
||||
from db import get_session
|
||||
from innercontext.api.utils import get_or_404
|
||||
from innercontext.llm import get_gemini_client
|
||||
from innercontext.models import (
|
||||
Product,
|
||||
ProductBase,
|
||||
|
|
@ -350,11 +349,7 @@ OUTPUT SCHEMA (all fields optional — omit what you cannot determine):
|
|||
|
||||
@router.post("/parse-text", response_model=ProductParseResponse)
|
||||
def parse_product_text(data: ProductParseRequest) -> ProductParseResponse:
|
||||
api_key = os.environ.get("GEMINI_API_KEY")
|
||||
if not api_key:
|
||||
raise HTTPException(status_code=503, detail="GEMINI_API_KEY not configured")
|
||||
model = os.environ.get("GEMINI_MODEL", "gemini-flash-latest")
|
||||
client = genai.Client(api_key=api_key)
|
||||
client, model = get_gemini_client()
|
||||
response = client.models.generate_content(
|
||||
model=model,
|
||||
contents=f"Extract product data from this text:\n\n{data.text}",
|
||||
|
|
|
|||
|
|
@ -1,12 +1,16 @@
|
|||
import json
|
||||
from datetime import date
|
||||
from typing import Optional
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from google.genai import types as genai_types
|
||||
from pydantic import ValidationError
|
||||
from sqlmodel import Session, SQLModel, select
|
||||
|
||||
from db import get_session
|
||||
from innercontext.api.utils import get_or_404
|
||||
from innercontext.llm import get_gemini_client
|
||||
from innercontext.models import (
|
||||
SkinConditionSnapshot,
|
||||
SkinConditionSnapshotBase,
|
||||
|
|
@ -51,11 +55,118 @@ class SnapshotUpdate(SQLModel):
|
|||
notes: Optional[str] = None
|
||||
|
||||
|
||||
class SkinPhotoAnalysisResponse(SQLModel):
|
||||
overall_state: Optional[OverallSkinState] = None
|
||||
trend: Optional[SkinTrend] = None
|
||||
skin_type: Optional[SkinType] = None
|
||||
hydration_level: Optional[int] = None
|
||||
sebum_tzone: Optional[int] = None
|
||||
sebum_cheeks: Optional[int] = None
|
||||
sensitivity_level: Optional[int] = None
|
||||
barrier_state: Optional[BarrierState] = None
|
||||
active_concerns: Optional[list[SkinConcern]] = None
|
||||
risks: Optional[list[str]] = None
|
||||
priorities: Optional[list[str]] = None
|
||||
notes: Optional[str] = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _skin_photo_system_prompt() -> str:
|
||||
return """\
|
||||
You are a dermatology-trained skin assessment AI. Analyze the provided photo(s) of a person's
|
||||
skin and return a structured JSON assessment.
|
||||
|
||||
RULES:
|
||||
- Return ONLY raw JSON — no markdown fences, no explanation.
|
||||
- Omit any field you cannot confidently determine from the photos. Do not guess.
|
||||
- All enum values must exactly match the allowed strings listed below.
|
||||
- Numeric metrics use a 1–5 scale (1 = minimal, 5 = maximal).
|
||||
- For trends: only populate if you have strong visual cues; otherwise omit.
|
||||
- risks and priorities: short English phrases, max 10 words each.
|
||||
- notes: 2–4 sentence paragraph describing key observations.
|
||||
|
||||
ENUM VALUES:
|
||||
overall_state: "excellent" | "good" | "fair" | "poor"
|
||||
trend: "improving" | "stable" | "worsening" | "fluctuating"
|
||||
skin_type: "dry" | "oily" | "combination" | "sensitive" | "normal" | "acne_prone"
|
||||
barrier_state: "intact" | "mildly_compromised" | "compromised"
|
||||
active_concerns: "acne" | "rosacea" | "hyperpigmentation" | "aging" | "dehydration" |
|
||||
"redness" | "damaged_barrier" | "pore_visibility" | "uneven_texture" | "sebum_excess"
|
||||
|
||||
METRICS (int 1–5, omit if not assessable):
|
||||
hydration_level: 1=very dehydrated/dull → 5=plump/luminous
|
||||
sebum_tzone: 1=very dry T-zone → 5=very oily T-zone
|
||||
sebum_cheeks: 1=very dry cheeks → 5=very oily cheeks
|
||||
sensitivity_level: 1=no visible signs → 5=severe redness/reactivity
|
||||
|
||||
OUTPUT (all fields optional):
|
||||
{"overall_state":…, "trend":…, "skin_type":…, "hydration_level":…,
|
||||
"sebum_tzone":…, "sebum_cheeks":…, "sensitivity_level":…,
|
||||
"barrier_state":…, "active_concerns":[…], "risks":[…], "priorities":[…], "notes":…}
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
MAX_IMAGE_BYTES = 5 * 1024 * 1024 # 5 MB
|
||||
|
||||
|
||||
@router.post("/analyze-photos", response_model=SkinPhotoAnalysisResponse)
|
||||
async def analyze_skin_photos(
|
||||
photos: list[UploadFile] = File(...),
|
||||
) -> SkinPhotoAnalysisResponse:
|
||||
if not (1 <= len(photos) <= 3):
|
||||
raise HTTPException(status_code=422, detail="Send between 1 and 3 photos.")
|
||||
|
||||
client, model = get_gemini_client()
|
||||
|
||||
allowed = {"image/jpeg", "image/png", "image/webp"}
|
||||
parts: list[genai_types.Part] = []
|
||||
for photo in photos:
|
||||
if photo.content_type not in allowed:
|
||||
raise HTTPException(status_code=422, detail=f"Unsupported type: {photo.content_type}")
|
||||
data = await photo.read()
|
||||
if len(data) > MAX_IMAGE_BYTES:
|
||||
raise HTTPException(status_code=413, detail=f"{photo.filename} exceeds 5 MB.")
|
||||
parts.append(genai_types.Part.from_bytes(data=data, mime_type=photo.content_type))
|
||||
parts.append(
|
||||
genai_types.Part.from_text(
|
||||
"Analyze the skin condition visible in the above photo(s) and return the JSON assessment."
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
response = client.models.generate_content(
|
||||
model=model,
|
||||
contents=parts,
|
||||
config=genai_types.GenerateContentConfig(
|
||||
system_instruction=_skin_photo_system_prompt(),
|
||||
response_mime_type="application/json",
|
||||
max_output_tokens=1024,
|
||||
temperature=0.0,
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"Gemini API error: {e}")
|
||||
|
||||
try:
|
||||
parsed = json.loads(response.text)
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(status_code=502, detail=f"LLM returned invalid JSON: {e}")
|
||||
|
||||
try:
|
||||
return SkinPhotoAnalysisResponse.model_validate(parsed)
|
||||
except ValidationError as e:
|
||||
raise HTTPException(status_code=422, detail=e.errors())
|
||||
|
||||
|
||||
@router.get("", response_model=list[SkinConditionSnapshotPublic])
|
||||
def list_snapshots(
|
||||
from_date: Optional[date] = None,
|
||||
|
|
|
|||
21
backend/innercontext/llm.py
Normal file
21
backend/innercontext/llm.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
"""Shared helpers for Gemini API access."""
|
||||
|
||||
import os
|
||||
|
||||
from fastapi import HTTPException
|
||||
from google import genai
|
||||
|
||||
|
||||
_DEFAULT_MODEL = "gemini-flash-latest"
|
||||
|
||||
|
||||
def get_gemini_client() -> tuple[genai.Client, str]:
|
||||
"""Return an authenticated Gemini client and the configured model name.
|
||||
|
||||
Raises HTTP 503 if GEMINI_API_KEY is not set.
|
||||
"""
|
||||
api_key = os.environ.get("GEMINI_API_KEY")
|
||||
if not api_key:
|
||||
raise HTTPException(status_code=503, detail="GEMINI_API_KEY not configured")
|
||||
model = os.environ.get("GEMINI_MODEL", _DEFAULT_MODEL)
|
||||
return genai.Client(api_key=api_key), model
|
||||
|
|
@ -9,6 +9,7 @@ dependencies = [
|
|||
"google-genai>=1.65.0",
|
||||
"psycopg>=3.3.3",
|
||||
"python-dotenv>=1.2.1",
|
||||
"python-multipart>=0.0.22",
|
||||
"sqlmodel>=0.0.37",
|
||||
"uvicorn[standard]>=0.34.0",
|
||||
]
|
||||
|
|
|
|||
11
backend/uv.lock
generated
11
backend/uv.lock
generated
|
|
@ -459,6 +459,7 @@ dependencies = [
|
|||
{ name = "google-genai" },
|
||||
{ name = "psycopg" },
|
||||
{ name = "python-dotenv" },
|
||||
{ name = "python-multipart" },
|
||||
{ name = "sqlmodel" },
|
||||
{ name = "uvicorn", extra = ["standard"] },
|
||||
]
|
||||
|
|
@ -479,6 +480,7 @@ requires-dist = [
|
|||
{ name = "google-genai", specifier = ">=1.65.0" },
|
||||
{ name = "psycopg", specifier = ">=3.3.3" },
|
||||
{ name = "python-dotenv", specifier = ">=1.2.1" },
|
||||
{ name = "python-multipart", specifier = ">=0.0.22" },
|
||||
{ name = "sqlmodel", specifier = ">=0.0.37" },
|
||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" },
|
||||
]
|
||||
|
|
@ -710,6 +712,15 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-multipart"
|
||||
version = "0.0.22"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/94/01/979e98d542a70714b0cb2b6728ed0b7c46792b695e3eaec3e20711271ca3/python_multipart-0.0.22.tar.gz", hash = "sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58", size = 37612, upload-time = "2026-01-25T10:15:56.219Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/d0/397f9626e711ff749a95d96b7af99b9c566a9bb5129b8e4c10fc4d100304/python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155", size = 24579, upload-time = "2026-01-25T10:15:54.811Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytokens"
|
||||
version = "0.4.1"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue