From 5bb2ea5f08c8660f8962056a47c8479aa57851d7 Mon Sep 17 00:00:00 2001 From: Piotr Oleszczyk Date: Fri, 6 Mar 2026 10:58:26 +0100 Subject: [PATCH] feat(api): add short_id column for consistent LLM UUID handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves validation failures where LLM fabricated full UUIDs from 8-char prefixes shown in context, causing 'unknown product_id' errors. Root Cause Analysis: - Context showed 8-char short IDs: '77cbf37c' (Phase 2 optimization) - Function tool returned full UUIDs: '77cbf37c-3830-4927-...' - LLM saw BOTH formats, got confused, invented UUIDs for final response - Validators rejected fabricated UUIDs as unknown products Solution: Consistent 8-char short_id across LLM boundary: 1. Database: New short_id column (8 chars, unique, indexed) 2. Context: Shows short_id (was: str(id)[:8]) 3. Function tools: Return short_id (was: full UUID) 4. Translation layer: Expands short_id → UUID before validation 5. Database: Stores full UUIDs (no schema change for existing data) Changes: - Added products.short_id column with unique constraint + index - Migration populates from UUID prefix, handles collisions via regeneration - Product model auto-generates short_id for new products - LLM contexts use product.short_id consistently - Function tools return product.short_id - Added _expand_product_id() translation layer in routines.py - Integrated expansion in suggest_routine() and suggest_batch() - Validators work with full UUIDs (no changes needed) Benefits: ✅ LLM never sees full UUIDs, no format confusion ✅ Maintains Phase 2 token optimization (~85% reduction) ✅ O(1) indexed short_id lookups vs O(n) pattern matching ✅ Unique constraint prevents collisions at DB level ✅ Clean separation: 8-char for LLM, 36-char for application From production error: Step 1: unknown product_id 77cbf37c-3830-4927-9669-07447206689d (LLM invented the last 28 characters) Now resolved: LLM uses '77cbf37c' consistently, translation layer expands to real UUID before validation. --- backend/.coverage | Bin 0 -> 53248 bytes ...306b0c6_add_short_id_column_to_products.py | 83 ++++++++++++++++++ backend/innercontext/api/llm_context.py | 2 +- backend/innercontext/api/product_llm_tools.py | 7 +- backend/innercontext/api/routines.py | 75 +++++++++++++--- backend/innercontext/models/product.py | 11 +++ backend/jobs/2026-03-02__17-12-31/job.log | 0 backend/pgloader.config | 12 +++ 8 files changed, 176 insertions(+), 14 deletions(-) create mode 100644 backend/.coverage create mode 100644 backend/alembic/versions/27b2c306b0c6_add_short_id_column_to_products.py create mode 100644 backend/jobs/2026-03-02__17-12-31/job.log create mode 100644 backend/pgloader.config diff --git a/backend/.coverage b/backend/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..da6d65fee8f4336fcafe711a4a100506570883c5 GIT binary patch literal 53248 zcmeI5du$xV9mjWX?{4q0yW{uy?Tvvr4t~T+8XhIYDRB}f;njqPXyLN9*YO44?dA4t zViIB>pi)5#f>Nr|RLNgOY9(4#3KRmtEmB)bRfCEs5SoNi#S}yUnW|0*x&CHu_Z~@% z&|W3v^tZC_w>vxY-QRrXH#0lF+w-RDH%5}WI%p)KT2fugm2f=Ib*n1Jaenxf;n&_o z*m2tjVEMfLxpw_r$Nd#Dxsj8+H*@4Rxt6p`TcuBed!$SKHv}VMo4*52;D!W{01`j~ z=M90>^^zE>sp0RwE2;Gj>&b-HuO~9E@>SpLS+%)G-Ms1>8++7Dnz|^U!q(odu2K`m z9co-psDqJVU5&&BBK=x265FaKhjcslh^Y_IISbNY)}k3ZSb4e^jtoGlq`nm-;)zI9 zON^tk&!gLSo}z9X4Q01-X1HAZu&i>%_UYC<2>6MC#)H#1%=iVU;{TJISTh@tXw z{yv@6NlWy@Pq3gikdUiN?;r^E8;JqxeP2S0^$+Q0o2q4N*l&QJJCd2&CX8rj(olyZ zF;>l(8A(Qrn5ys4`$v-cz&R_}Mxogoz{{@?!8dhb$blaSxEpzoPXighBZP84r zkyvE=h@NfyHZ@nWja4INw5hZb5CInjy`js>c@Cmd-(hYahFhnwHj*^haTppu+yyVv zy`?@e)YQb^x19wF#S`v}B+cxOzaVI`$#aAbJwJEc&|Xp)B(xU|539x?>oT;onrR$} z=KMa>X~WPGw4=8|rDAD|E6U5bnzE22wR%F;rK_-|c5%WVM!M#DWmATk;&k zsC_D`YvxG89!S1ukhDwWMoGJmSZAO*?V>lds&208Soy+T){ZYFy2VgsC7;Ul2s$p& zUr#|LXiBaNoG}r6dH3U5a!5VfcbW{2w|KmvuUF3HICUZ1MPI%5mbjS9-xV1BC=B}a zS153qrsQ1SppUhWYQr#^#I;BQx-5*rnHfN(83#1Tto0cqNfy8KkcB6mdvg9}1aikV zd&t$wvP1X2&M?ggOsjO-GB3AN)s`X6q@$48qV^fau&%{2a|4vYVguHEMW^A6DvbTKbJFW%1`#Nzwm_#ep^KiO zX^UEC>4;_Pcf^4SRgG!`neI*-B~S5=A^j}5%xtt;OX|r;R99&h<_rt0J2X@6Pv~&c znoF5uKy2ttuQD?0;EX+nW@^*gB79No&?XLd(O0@k_R(9pR1+_TF20ygjaWT6GdP7! zxD$wKc3rJQS3zW*aWWTV*TN_bw@CTHMS1q4ZQtl09Qb$^ILxfpzr@TdrooWH)K0m zsQg`dOu1U|$j`|Md69HjdQ|F_gy6y8NU%NdkHEu$O##_I=^yhi^1bPM(08eLRJ>no z^ZwQQJ@0A|!3_x@0VIF~<^_Q@&2BDL+IfQs59f8pBStdOIiQbrMq)8NVLiCn*{Aj2 zuEz!n6Sa7x)4V+rgXiCMcu;T6f+BPh@HixWz8Q3SRL~&~{fQ?G_Jm-3?4~A=S?WLr zp1h2}lYR!!Fo%qI93I;bXVbvHZ#RPQisFR9&4H2rWH_!ROuayTa|5Wi7pXpEB#i>q z8|y)}t2kAwi3)T#*A?$=pO)+&Dp2jM1=Ub--u7uG4a%!)K&q)Ysc<+FizLHgkh``T zAGUlsiB~U5dm;^%x9N6!ldEgn|x*VBBSZRx2Rlq}8c6L%KGc9HN+IP;r7e zY7FSZFyiLslXVhERXV7ZC1s6Q;MBTcaZ0&KWnG|XB|7d?pZvutv2g^H*7`unNf=q2 zvTe6U1R*DJZ3*e{Wg9H6y~Q~+U_`Y@3~JWn0U;-Yqven_2ds4$=Z_W%4;x#-pI!lk ziU{t!F}BC$uxhloWgMcY*YF_Yq{*mEo;&pY|7OqU9C?N$BzJH$@NVFdV2Ahn-X72A z!50FX0}5$T4l6%Wu2H=5^Rg+oN=Kw0O6&c<_221l^}XeL$ak6efY>g+<9%&j-1MT8 zNB{{S0VKe2Yu%|*-obqqokQ~O*P4sb$y@&~SmRC+r~c$_)zq8asijV2vTOZ0*8fen zx>GBP6t-6=XRrSoZ*ZsDi&Z~s{ointJJnUB>bchc_5TBJ&tCu6t#_wFMS5%9`_*3K zhHqbT5ZrlDkawR{bDcYNffKpxa{Y|;f3?$K&kC~je`upSRacZ>d6lf{ai?S_Vg>8} z$`!)6LlL;Kt#AsO4oY_K%3uGNJB=wSSh)T#D`udxe4{t-rR&_OawmwWOZn^nlGW~1 z(5aBU{wGdboqCg9Ln=-&TT8aQ`9$t@rz)M)%96^h|D__8a+6B1cqME7A1GSMUjO@@ zhLOc7+jc&uacv3N>wmFGhqCK`uhYTNa>$wkJVpAGUH`j_4DP%!R&ZK1+S_vLe;3G< z`W;MfR3^_I{Qf^&2p|C@fCP{L51q(c4GACt zB!C2v01`j~NB{{Sf%AmGf@U}GU)}WV+sn&2uA7V7)jG;4usb1e`$*dY$Z|{lozFxM zE&qVy_RTIl(9z3XGPdxgPaZ%1>hdREKlYkY{?5O?B z(nNQT(w9$z4Unt4_VGhc{rq<>u^tXeYEF7e>tN@?>TL=K|F_-8so;lF3rUTkgFp4b z^?h8ow+4=tD!!x|Y_t4q!y98S@h@I<$X!oDkfKVl(8|LrAKZC7Dg>$^p(+@gK5?E}XJS(sq;`w0si~Fez0*_anRNQZ)XKeNc4lT` zW@2iZ_m)Bi#hdV!Q0plR60p_on3*0oLqUZe2+(TEQB{%Qps{j2yE@OQH4+>zt2o)w z#O(qU@3VCJGI6IEgi|Ht7FSCEcCKqa_|aAST*r2?c>nT952>Kd7;ZD6K`t>XFWI(8JQWu~fz8M&Gn zDa1^$ikUzqGyV!@eC5oDWz2X>nemh`<0i}q3Nw{5GZhjuB*;v2fSCn;W>gE7NB!C2v01`j~ SNB{{S0VIF~kN^^JNZ`Mn;oAHF literal 0 HcmV?d00001 diff --git a/backend/alembic/versions/27b2c306b0c6_add_short_id_column_to_products.py b/backend/alembic/versions/27b2c306b0c6_add_short_id_column_to_products.py new file mode 100644 index 0000000..8f4cde4 --- /dev/null +++ b/backend/alembic/versions/27b2c306b0c6_add_short_id_column_to_products.py @@ -0,0 +1,83 @@ +"""add short_id column to products + +Revision ID: 27b2c306b0c6 +Revises: 2697b4f1972d +Create Date: 2026-03-06 10:54:13.308340 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "27b2c306b0c6" +down_revision: Union[str, Sequence[str], None] = "2697b4f1972d" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema. + + Add short_id column (8-char prefix of UUID) for LLM token optimization. + Handles collisions by regenerating conflicting short_ids. + """ + # Step 1: Add column (nullable initially) + op.add_column("products", sa.Column("short_id", sa.String(8), nullable=True)) + + # Step 2: Populate from existing UUIDs with collision detection + connection = op.get_bind() + + # Get all products + result = connection.execute(sa.text("SELECT id FROM products")) + products = [(str(row[0]),) for row in result] + + # Track used short_ids to detect collisions + used_short_ids = set() + + for (product_id,) in products: + short_id = product_id[:8] + + # Handle collision: regenerate using next 8 chars, or random + if short_id in used_short_ids: + # Try using chars 9-17 + alternative = product_id[9:17] if len(product_id) > 16 else None + if alternative and alternative not in used_short_ids: + short_id = alternative + else: + # Generate random 8-char hex + import secrets + + while True: + short_id = secrets.token_hex(4) # 8 hex chars + if short_id not in used_short_ids: + break + + print(f"COLLISION RESOLVED: UUID {product_id} → short_id {short_id}") + + used_short_ids.add(short_id) + + # Update product with short_id + connection.execute( + sa.text("UPDATE products SET short_id = :short_id WHERE id = :id"), + {"short_id": short_id, "id": product_id}, + ) + + # Step 3: Add NOT NULL constraint + op.alter_column("products", "short_id", nullable=False) + + # Step 4: Add unique constraint + op.create_unique_constraint("uq_products_short_id", "products", ["short_id"]) + + # Step 5: Add index for fast lookups + op.create_index("idx_products_short_id", "products", ["short_id"]) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index("idx_products_short_id", table_name="products") + op.drop_constraint("uq_products_short_id", "products", type_="unique") + op.drop_column("products", "short_id") diff --git a/backend/innercontext/api/llm_context.py b/backend/innercontext/api/llm_context.py index 6b9b19e..6ffc68e 100644 --- a/backend/innercontext/api/llm_context.py +++ b/backend/innercontext/api/llm_context.py @@ -117,7 +117,7 @@ def build_product_context_summary(product: Product, has_inventory: bool = False) safety_str = f" safety={{{','.join(safety_flags)}}}" if safety_flags else "" return ( - f"{status} {str(product.id)[:8]} | {product.brand} {product.name} " + f"{status} {product.short_id} | {product.brand} {product.name} " f"({product.category}){effects_str}{safety_str}" ) diff --git a/backend/innercontext/api/product_llm_tools.py b/backend/innercontext/api/product_llm_tools.py index 44f03aa..e152fd6 100644 --- a/backend/innercontext/api/product_llm_tools.py +++ b/backend/innercontext/api/product_llm_tools.py @@ -96,9 +96,12 @@ def _map_product_details( The 128-ingredient INCI list was consuming ~15KB per product. For safety/clinical decisions, actives + effect_profile are sufficient. + Uses short_id (8 chars) for LLM consistency - translation layer expands + to full UUID before validation/database storage. + Args: product: Product to map - pid: Product ID string + pid: Product short_id (8 characters, e.g., "77cbf37c") last_used_on: Last usage date include_inci: Whether to include full INCI list (default: False) @@ -193,7 +196,7 @@ def build_product_details_tool_handler( products_payload.append( _map_product_details( product, - full_id, # Always use full ID in response + product.short_id, # Return short_id for LLM consistency last_used_on=last_used_on_by_product.get(full_id), ) ) diff --git a/backend/innercontext/api/routines.py b/backend/innercontext/api/routines.py index c97fc71..7134beb 100644 --- a/backend/innercontext/api/routines.py +++ b/backend/innercontext/api/routines.py @@ -400,6 +400,42 @@ def _get_products_with_inventory( return set(inventory_rows) +def _expand_product_id(session: Session, short_or_full_id: str) -> UUID | None: + """ + Expand 8-char short_id to full UUID, or validate full UUID. + + Translation layer between LLM world (8-char short_ids) and application world + (36-char UUIDs). LLM sees/uses short_ids for token optimization, but + validators and database use full UUIDs. + + Args: + session: Database session + short_or_full_id: Either short_id ("77cbf37c") or full UUID + + Returns: + Full UUID if product exists, None otherwise + """ + # Already a full UUID? + if len(short_or_full_id) == 36: + try: + uuid_obj = UUID(short_or_full_id) + # Verify it exists + product = session.get(Product, uuid_obj) + return uuid_obj if product else None + except (ValueError, TypeError): + return None + + # Short ID (8 chars) - indexed lookup + if len(short_or_full_id) == 8: + product = session.exec( + select(Product).where(Product.short_id == short_or_full_id) + ).first() + return product.id if product else None + + # Invalid length + return None + + def _build_objectives_context(include_minoxidil_beard: bool) -> str: if include_minoxidil_beard: return ( @@ -686,17 +722,26 @@ def suggest_routine( except json.JSONDecodeError as e: raise HTTPException(status_code=502, detail=f"LLM returned invalid JSON: {e}") - steps = [ - SuggestedStep( - product_id=UUID(s["product_id"]) if s.get("product_id") else None, - action_type=s.get("action_type") or None, - action_notes=s.get("action_notes"), - region=s.get("region"), - why_this_step=s.get("why_this_step"), - optional=s.get("optional"), + # Translation layer: Expand short_ids (8 chars) to full UUIDs (36 chars) + steps = [] + for s in parsed.get("steps", []): + product_id_str = s.get("product_id") + product_id_uuid = None + + if product_id_str: + # Expand short_id or validate full UUID + product_id_uuid = _expand_product_id(session, product_id_str) + + steps.append( + SuggestedStep( + product_id=product_id_uuid, + action_type=s.get("action_type") or None, + action_notes=s.get("action_notes"), + region=s.get("region"), + why_this_step=s.get("why_this_step"), + optional=s.get("optional"), + ) ) - for s in parsed.get("steps", []) - ] summary_raw = parsed.get("summary") or {} confidence_raw = summary_raw.get("confidence", 0) @@ -854,11 +899,19 @@ def suggest_batch( raise HTTPException(status_code=502, detail=f"LLM returned invalid JSON: {e}") def _parse_steps(raw_steps: list) -> list[SuggestedStep]: + """Parse steps and expand short_ids to full UUIDs.""" result = [] for s in raw_steps: + product_id_str = s.get("product_id") + product_id_uuid = None + + if product_id_str: + # Translation layer: expand short_id to full UUID + product_id_uuid = _expand_product_id(session, product_id_str) + result.append( SuggestedStep( - product_id=UUID(s["product_id"]) if s.get("product_id") else None, + product_id=product_id_uuid, action_type=s.get("action_type") or None, action_notes=s.get("action_notes"), region=s.get("region"), diff --git a/backend/innercontext/models/product.py b/backend/innercontext/models/product.py index c6c4a81..db4ed15 100644 --- a/backend/innercontext/models/product.py +++ b/backend/innercontext/models/product.py @@ -142,6 +142,12 @@ class Product(ProductBase, table=True): __domains__: ClassVar[frozenset[Domain]] = frozenset({Domain.SKINCARE}) id: UUID = Field(default_factory=uuid4, primary_key=True) + short_id: str = Field( + max_length=8, + unique=True, + index=True, + description="8-character short ID for LLM contexts (first 8 chars of UUID)", + ) # Override 9 JSON fields with sa_column (only in table model) inci: list[str] = Field( @@ -214,6 +220,11 @@ class Product(ProductBase, table=True): if self.price_currency is not None: self.price_currency = self.price_currency.upper() + # Auto-generate short_id from UUID if not set + # Migration handles existing products; this is for new products + if not hasattr(self, "short_id") or not self.short_id: + self.short_id = str(self.id)[:8] + return self def to_llm_context( diff --git a/backend/jobs/2026-03-02__17-12-31/job.log b/backend/jobs/2026-03-02__17-12-31/job.log new file mode 100644 index 0000000..e69de29 diff --git a/backend/pgloader.config b/backend/pgloader.config new file mode 100644 index 0000000..877cd1b --- /dev/null +++ b/backend/pgloader.config @@ -0,0 +1,12 @@ +LOAD DATABASE + FROM postgresql://innercontext_user:dpeBM6P79CZovjLKQdXc@192.168.101.83/innercontext + INTO sqlite:///Users/piotr/dev/innercontext/backend/innercontext.db + + WITH include drop, + create tables, + create indexes, + reset sequences + + SET work_mem to '16MB', + maintenance_work_mem to '512 MB'; +