US-09: Token Budget Allocation — Context Window Partitioning

graph TD
    subgraph "Inference Team"
        I["Give the LLM everything:<br/>20 turns of history,<br/>5 RAG chunks,<br/>full product data,<br/>active promotions.<br/>More context = better answers."]
    end

    subgraph "Performance Team"
        P["Every 100 tokens adds<br/>~10ms to prefill latency.<br/>A 4K token prompt takes<br/>400ms just to process.<br/>Keep it under 2K tokens."]
    end

    subgraph "Cost Team"
        C["Input tokens cost $3/1M.<br/>At 4K tokens × 700K LLM calls/day,<br/>that's $8,400/day in input alone.<br/>Cut to 2K tokens = $4,200/day.<br/>Save $126K/month."]
    end

    I ---|"Latency<br/>concern"| P
    P ---|"Cost<br/>impact"| C
    C ---|"Quality<br/>risk"| I

    style I fill:#ff6b6b,stroke:#333,color:#000
    style P fill:#4ecdc4,stroke:#333,color:#000
    style C fill:#f9d71c,stroke:#333,color:#000

graph TD
    subgraph "Theoretical Max: 200K tokens"
        THEORY["Claude 3.5 Sonnet<br/>can process 200K tokens.<br/>But at $3/1M input tokens,<br/>200K = $0.60 per request.<br/>At 700K requests/day = $420K/day.<br/>❌ Budget: destroyed."]
    end

    subgraph "Practical Budget: 2,000 tokens"
        PRACTICAL["2,000 input tokens<br/>300 output tokens<br/>Cost: $0.006 + $0.0045 = $0.0105<br/>Prefill: ~200ms<br/>✅ Within budget and latency."]
    end

    THEORY --> PRACTICAL

    style THEORY fill:#eb3b5a,stroke:#333,color:#fff
    style PRACTICAL fill:#2d8659,stroke:#333,color:#fff

pie title "Token Budget Allocation (2,000 tokens)"
    "System Prompt" : 350
    "Conversation History" : 400
    "RAG Context" : 600
    "Current Data (product, promo)" : 400
    "User Message" : 150
    "Formatting / Separators" : 100

graph TD
    subgraph "System Prompt Contents"
        S1["Persona definition<br/>(50 tokens)"]
        S2["Core rules<br/>(150 tokens)"]
        S3["Output format specs<br/>(80 tokens)"]
        S4["Safety instructions<br/>(70 tokens)"]
    end

    S1 --> TOTAL["350 tokens total"]
    S2 --> TOTAL
    S3 --> TOTAL
    S4 --> TOTAL

    style TOTAL fill:#54a0ff,stroke:#333,color:#000

graph TD
    subgraph "Option A: Full History (Inference Team)"
        A["Keep all 10 turns raw<br/>~1,500 tokens<br/>✅ Perfect context<br/>❌ Blows budget by 1,100 tokens"]
    end

    subgraph "Option B: Last 2 Turns Only (Cost Team)"
        B["Only last user + assistant turn<br/>~200 tokens<br/>✅ Very cheap<br/>❌ Loses multi-turn context<br/>❌ 'What about the second one?'<br/>fails"]
    end

    subgraph "Option C: Summary + Last 2 (Decision)"
        C["Compressed summary of turns 1-8<br/>(~150 tokens)<br/>+<br/>Full last 2 turns (~250 tokens)<br/>= 400 tokens<br/>✅ Preserves key context<br/>✅ Within budget"]
    end

    style A fill:#eb3b5a,stroke:#333,color:#fff
    style B fill:#f9d71c,stroke:#333,color:#000
    style C fill:#2d8659,stroke:#333,color:#fff

sequenceDiagram
    participant Memory as Conversation Memory
    participant Summarizer as Summarizer (Haiku)
    participant Prompt as Prompt Builder

    Memory->>Prompt: Retrieve all turns for session

    alt Turn count ≤ 4
        Prompt->>Prompt: Include all turns verbatim (~400 tokens)
    else Turn count > 4
        Memory->>Summarizer: Summarize turns 1 through N-2
        Summarizer-->>Memory: Summary (150 tokens)
        Note over Summarizer: Cost: $0.0002 per summary<br/>(Haiku, ~400 input tokens)
        Prompt->>Prompt: Summary + last 2 full turns (~400 tokens)
    end

graph TD
    A["3 chunks retrieved"] --> B{"Total chunk<br/>tokens?"}
    B -->|"< 600"| C["Include all chunks<br/>verbatim"]
    B -->|"> 600"| D["Compress: extract<br/>key sentences only"]
    B -->|"Chunks irrelevant<br/>(low similarity)"| E["Include 0 chunks<br/>+ flag: ungrounded response"]

    style C fill:#2d8659,stroke:#333,color:#fff
    style D fill:#fd9644,stroke:#333,color:#000
    style E fill:#eb3b5a,stroke:#333,color:#fff

graph TD
    A["Total Budget: 2,000 tokens"] --> B["Fixed: System (350) + Format (100) = 450"]
    A --> C["Variable Pool: 1,550 tokens"]

    C --> D{"Intent Type"}
    D -->|"FAQ"| E["History: 500<br/>RAG: 800<br/>Data: 100<br/>Message: 150"]
    D -->|"Recommendation"| F["History: 400<br/>RAG: 400<br/>Data: 600<br/>Message: 150"]
    D -->|"Product Question"| G["History: 300<br/>RAG: 500<br/>Data: 600<br/>Message: 150"]
    D -->|"Order Tracking"| H["History: 500<br/>RAG: 0<br/>Data: 300<br/>Message: 150<br/>(+600 unallocated → shorter prompt)"]

    style E fill:#54a0ff,stroke:#333,color:#000
    style F fill:#fd9644,stroke:#333,color:#000
    style G fill:#ff6b6b,stroke:#333,color:#000
    style H fill:#2d8659,stroke:#333,color:#fff

graph LR
    A["1. Trim conversation<br/>history (summarize<br/>more aggressively)"] --> B["2. Reduce RAG<br/>chunks (from 3→2)"]
    B --> C["3. Compress product<br/>data (key fields only)"]
    C --> D["4. Never trim<br/>user message"]
    D --> E["5. Never trim<br/>system prompt"]

    style A fill:#f9d71c,stroke:#333,color:#000
    style D fill:#eb3b5a,stroke:#333,color:#fff
    style E fill:#eb3b5a,stroke:#333,color:#fff

graph TD
    A["Request 1:<br/>System Prompt (350 tokens)<br/>+ Conversation (400)<br/>+ RAG (600)<br/>+ Data (400)<br/>+ Message (150)"]

    B["Request 2 (same session):<br/>System Prompt (CACHED — 0 new tokens)<br/>+ Conversation (400)<br/>+ RAG (600)<br/>+ Data (400)<br/>+ Message (150)"]

    A --> C["Bedrock caches the<br/>system prompt prefix<br/>across requests"]
    C --> B

    style C fill:#2d8659,stroke:#333,color:#fff