US-08: Autoscaling Strategy — Cost vs Performance Headroom

graph TD
    subgraph "Performance Team"
        P["Keep 50% headroom at all times.<br/>Autoscaling is reactive — by the<br/>time new tasks spin up (60-90s),<br/>users have already seen timeouts.<br/>Pre-provision for peak."]
    end

    subgraph "Cost Team"
        C["50% headroom means paying for<br/>containers that sit idle 70% of<br/>the time. At $0.04/vCPU-hour,<br/>that's $28K/month in waste.<br/>Scale tighter, accept occasional<br/>latency spikes."]
    end

    subgraph "Inference Team"
        I["If tasks are overloaded,<br/>LLM requests queue up.<br/>Queued requests = stale context<br/>(user may have navigated away).<br/>Overloaded tasks also drop<br/>WebSocket connections."]
    end

    P ---|"Idle cost"| C
    C ---|"Quality under<br/>load"| I
    I ---|"Capacity<br/>planning"| P

    style P fill:#4ecdc4,stroke:#333,color:#000
    style C fill:#f9d71c,stroke:#333,color:#000
    style I fill:#ff6b6b,stroke:#333,color:#000

graph LR
    subgraph "Traffic Pattern — Typical Weekday"
        direction TB
        T1["6AM: Wake-up ramp<br/>200 RPS"]
        T2["12PM: Lunch spike<br/>800 RPS"]
        T3["3PM: Afternoon lull<br/>500 RPS"]
        T4["7PM: Evening peak<br/>1,500 RPS"]
        T5["10PM: Wind-down<br/>600 RPS"]
        T6["1AM: Off-peak<br/>50 RPS"]
    end

    T1 --> T2 --> T3 --> T4 --> T5 --> T6

    style T4 fill:#eb3b5a,stroke:#333,color:#fff
    style T6 fill:#2d8659,stroke:#333,color:#fff

graph TD
    subgraph "6:30 PM — Traffic starts climbing"
        A1["Current: 600 RPS<br/>Capacity: 800 RPS<br/>Headroom: 33%"]
    end

    subgraph "7:00 PM — Spike hits"
        A2["Demand: 1,500 RPS<br/>Capacity: 800 RPS<br/>Deficit: 700 RPS ❌"]
    end

    subgraph "7:02 PM — Autoscaler triggers"
        A3["Scaling event fired<br/>New tasks launching...<br/>Time to ready: 60-90s"]
    end

    subgraph "7:03 PM — Gap period"
        A4["700 RPS queued or dropped<br/>Latency spikes to 5-8 seconds<br/>15% of users see errors"]
    end

    A1 --> A2 --> A3 --> A4

    style A2 fill:#eb3b5a,stroke:#333,color:#fff
    style A4 fill:#ff6b6b,stroke:#333,color:#000

graph TD
    A["Monitor: CPU > 70%<br/>for 2 minutes"] --> B["Scale up:<br/>add 20% capacity"]
    B --> C["Tasks ready<br/>in 60-90 seconds"]
    C --> D["Gap: 2-3 minutes<br/>of degraded performance"]

    style D fill:#eb3b5a,stroke:#333,color:#fff

graph TD
    A["Always maintain:<br/>50% headroom above<br/>current baseline"] --> B["No scaling gap:<br/>capacity always available"]

    style B fill:#2d8659,stroke:#333,color:#fff

graph TD
    A["Predictive Scaling<br/>(scheduled)"] --> B["Pre-provision for known<br/>patterns:<br/>• Evening peak at 6:30 PM<br/>• Lunch spike at 11:30 AM<br/>• Manga release days"]

    C["Reactive Scaling<br/>(metric-based)"] --> D["Handle unexpected spikes:<br/>• Viral recommendation<br/>• Flash sale<br/>• External event"]

    B --> E["Combined: Capacity ready<br/>before predictable spikes<br/>+ reactive for surprises"]
    D --> E

    E --> F["Lambda Burst Pool<br/>(overflow)"] 

    style B fill:#2d8659,stroke:#333,color:#fff
    style D fill:#fd9644,stroke:#333,color:#000
    style F fill:#f9d71c,stroke:#333,color:#000

sequenceDiagram
    participant Scheduler as Predictive Scheduler
    participant ASG as ECS Auto Scaling
    participant Fargate as Fargate Tasks
    participant Lambda as Lambda Burst Pool
    participant CW as CloudWatch

    Note over Scheduler: 6:15 PM JST — 15 min before predicted peak
    Scheduler->>ASG: Scale to peak capacity (1,500 RPS)
    ASG->>Fargate: Launch additional tasks
    Note over Fargate: Tasks ready by 6:25 PM

    Note over CW: 7:00 PM — Actual peak arrives
    CW->>ASG: CPU at 65% (within target)
    Note over Fargate: Handling peak without scaling gap ✅

    Note over CW: 7:30 PM — Unexpected viral spike (+500 RPS)
    CW->>ASG: CPU at 85% — reactive scale-up
    ASG->>Fargate: Launch more tasks (60s warm-up)
    CW->>Lambda: Overflow to Lambda burst pool (instant)
    Note over Lambda: Lambda handles overflow during 60s gap

graph TD
    A["Request arrives"] --> B{"Fargate tasks<br/>available?"}
    B -->|"Yes"| C["Route to Fargate<br/>(preferred — WebSocket,<br/>stateful)"]
    B -->|"No — at capacity"| D["Route to Lambda<br/>(overflow — HTTPS only,<br/>stateless)"]

    D --> E["Lambda limitations:<br/>• No WebSocket streaming<br/>• Stateless (no session affinity)<br/>• Cold start: 200-500ms<br/>• Concurrency limit: 1,000"]

    style C fill:#2d8659,stroke:#333,color:#fff
    style D fill:#fd9644,stroke:#333,color:#000

graph TD
    subgraph "On-Demand Bedrock"
        OD["Pay per token<br/>No capacity guarantee<br/>Latency: 200-800ms TTFT<br/>Risk: throttling at peak"]
    end

    subgraph "Provisioned Throughput"
        PT["Reserved capacity<br/>$25K/month base<br/>Latency: 150-400ms TTFT<br/>Guaranteed throughput"]
    end

    subgraph "Hybrid (Decision)"
        HY["Provisioned for peak hours<br/>(8AM-11PM JST)<br/>On-demand for off-peak<br/>Saves ~$10K/month vs 24/7"]
    end

    style OD fill:#2d8659,stroke:#333,color:#fff
    style PT fill:#eb3b5a,stroke:#333,color:#fff
    style HY fill:#fd9644,stroke:#333,color:#000

graph TD
    A["Traffic decreasing"] --> B{"CPU < 40%<br/>for 10 minutes?"}
    B -->|"Yes"| C["Scale down by<br/>10% of current capacity"]
    B -->|"No"| D["Hold current capacity"]

    C --> E{"Minimum floor<br/>reached?"}
    E -->|"Yes"| F["Stop scaling down"]
    E -->|"No"| A

    style C fill:#f9d71c,stroke:#333,color:#000
    style F fill:#eb3b5a,stroke:#333,color:#fff