US-04: Real-Time vs Pre-Computed Inference

graph TD
    subgraph "Performance Team"
        P["Pre-compute everything!<br/>Real-time inference is slow.<br/>If we pre-generate top 100<br/>recommendations per user,<br/>response time drops to<br/>a cache lookup."]
    end

    subgraph "Inference Team"
        I["Pre-computed results go stale.<br/>The user's current page context<br/>and conversation turn matter.<br/>You can't pre-compute a response to<br/>a question nobody asked yet."]
    end

    subgraph "Cost Team"
        C["Batch inference is 3-5x cheaper<br/>per unit than real-time.<br/>But storing and refreshing<br/>pre-computed results costs money too.<br/>What's the break-even point?"]
    end

    P ---|"Staleness<br/>concern"| I
    I ---|"Batch cost vs<br/>real-time cost"| C
    C ---|"Latency<br/>SLA"| P

    style P fill:#4ecdc4,stroke:#333,color:#000
    style I fill:#ff6b6b,stroke:#333,color:#000
    style C fill:#f9d71c,stroke:#333,color:#000

graph LR
    subgraph "Fully Pre-Computed"
        A["Generated hours/days ahead<br/>Stored in cache/DB<br/>Nearest-neighbor lookup<br/>at request time"]
    end

    subgraph "Near-Real-Time"
        B["Pre-computed base<br/>+ lightweight real-time<br/>refinement at request time"]
    end

    subgraph "Fully Real-Time"
        C["Computed from scratch<br/>at request time<br/>using full conversation<br/>context"]
    end

    A -->|"Freshness<br/>decreases"| B
    B -->|"Latency<br/>increases"| C

    style A fill:#2d8659,stroke:#333,color:#fff
    style B fill:#fd9644,stroke:#333,color:#000
    style C fill:#eb3b5a,stroke:#333,color:#fff

graph TD
    subgraph "Batch Pipeline (runs every 6 hours)"
        B1["User purchase history"] --> B2["Collaborative filtering<br/>model inference"]
        B3["Browsing signals"] --> B2
        B2 --> B4["Top 50 candidate ASINs<br/>per user"]
        B4 --> B5["Store in DynamoDB<br/>key: user_id"]
    end

    subgraph "Real-Time Pipeline (at request time)"
        R1["Load pre-computed<br/>50 candidates"] --> R2["Filter by<br/>current context"]
        R3["Current page ASIN<br/>+ conversation history"] --> R2
        R2 --> R4["Rerank with<br/>lightweight model"]
        R4 --> R5["Top 5 final<br/>recommendations"]
        R5 --> R6["Feed to LLM<br/>for natural language"]
    end

    B5 -.->|"cache lookup<br/>5ms"| R1

    style B2 fill:#2d8659,stroke:#333,color:#fff
    style R4 fill:#fd9644,stroke:#333,color:#000
    style R6 fill:#eb3b5a,stroke:#333,color:#fff

graph TD
    subgraph "Batch: Pre-compile top 50 FAQs"
        F1["Analyze query logs<br/>→ top 50 FAQ questions"] --> F2["Generate Haiku answers<br/>for each variation"]
        F2 --> F3["Store in ElastiCache<br/>key: faq_question_hash"]
    end

    subgraph "Real-Time: Serve or generate"
        R1["User FAQ question"] --> R2{"Semantic similarity<br/>to pre-compiled FAQ<br/>> 0.92?"}
        R2 -->|"Yes"| R3["Return pre-compiled<br/>answer (5ms)"]
        R2 -->|"No"| R4["Full RAG + LLM<br/>pipeline (1,200ms)"]
    end

    F3 -.-> R2

    style R3 fill:#2d8659,stroke:#333,color:#fff
    style R4 fill:#eb3b5a,stroke:#333,color:#fff

graph TD
    A["Pre-computed answer:<br/>'Return window is 30 days'"] --> B{"Policy changed<br/>to 14 days"}
    B --> C["Stale answer served<br/>for up to 6 hours"]
    C --> D["Customer returns after<br/>day 15, gets denied"]
    D --> E["Support escalation<br/>+ trust damage"]

    style C fill:#eb3b5a,stroke:#333,color:#fff
    style E fill:#ff6b6b,stroke:#333,color:#000

graph LR
    subgraph "Pre-Computation Cost"
        PC1["Batch compute: run model<br/>on N items"] --> PC2["Storage: store results<br/>in cache/DB"]
        PC2 --> PC3["Refresh: re-run periodically"]
        PC3 --> PC4["Total = batch_cost +<br/>storage + refresh_cost"]
    end

    subgraph "Real-Time Cost"
        RT1["Per-request compute:<br/>model inference"] --> RT2["Scales linearly<br/>with traffic"]
        RT2 --> RT3["Total = cost_per_request<br/>× request_volume"]
    end

    PC4 --> BEP{"Break Even<br/>Point"}
    RT3 --> BEP

    style BEP fill:#54a0ff,stroke:#333,color:#000