FM System Performance — Scenarios and Runbooks

graph TB
    A["CloudWatch Alarm fires:<br/>p95 latency > 2000ms<br/>for 3 consecutive minutes"]
    B["On-call receives PagerDuty alert"]
    C{"Check X-Ray traces<br/>for high-latency requests"}
    D["X-Ray shows: bedrock-invoke<br/>segment has 1,800ms spikes<br/>(normally 350-800ms)"]
    E{"Drill into bedrock-invoke<br/>subsegments"}
    F["network-transit subsegment<br/>shows 1,200ms<br/>(normally 5-15ms)"]
    G{"Check DNS resolution<br/>timing in subsegment metadata"}
    H["DNS resolution for<br/>bedrock-runtime.ap-northeast-1.amazonaws.com<br/>taking 1,000-1,200ms<br/>(normally < 5ms)"]

    A --> B --> C --> D --> E --> F --> G --> H

    style A fill:#e74c3c,color:#fff
    style H fill:#e74c3c,color:#fff

# Shift traffic away from affected AZ
aws ecs update-service \
  --cluster manga-assist-prod \
  --service manga-assist-orchestrator \
  --placement-constraints "type=memberOf,expression=attribute:ecs.availability-zone != ap-northeast-1a" \
  --desired-count 8

graph TB
    P1["Enable DNS resolution<br/>monitoring per AZ"]
    P2["Add application-level<br/>DNS caching (60s TTL)"]
    P3["Configure DNS failover:<br/>if private zone fails,<br/>use cached result<br/>not public resolver"]
    P4["Deploy ECS tasks<br/>across 3 AZs<br/>(add ap-northeast-1d)"]
    P5["Add X-Ray annotation<br/>for AZ on every trace"]
    P6["Create CloudWatch alarm:<br/>DNS resolution > 50ms<br/>per AZ per service"]

    P1 --> P6
    P2 --> P3
    P4 --> P5

    style P1 fill:#2ecc71,color:#000
    style P2 fill:#2ecc71,color:#000
    style P3 fill:#2ecc71,color:#000
    style P4 fill:#2ecc71,color:#000
    style P5 fill:#2ecc71,color:#000
    style P6 fill:#2ecc71,color:#000

graph TB
    A["CloudWatch Alarm fires:<br/>Error rate > 5%<br/>for 2 consecutive minutes"]
    B["Check ECS task metrics:<br/>CPU 45%, Memory 60%<br/>— NOT resource-constrained"]
    C["Check Bedrock throttling:<br/>ThrottlingException count = 0<br/>— NOT Bedrock throttle"]
    D{"Check application logs<br/>for error patterns"}
    E["Logs show:<br/>ConnectionPoolExhausted<br/>'Unable to acquire connection<br/>to bedrock-runtime<br/>within 5000ms timeout'"]
    F{"Check connection pool metrics"}
    G["Pool size: 10 per task<br/>Active: 10/10 (100%)<br/>Queue depth: 45 waiting<br/>Avg checkout wait: 4,800ms"]

    A --> B --> C --> D --> E --> F --> G

    style A fill:#e74c3c,color:#fff
    style E fill:#e74c3c,color:#fff
    style G fill:#e74c3c,color:#fff

aws ecs update-service \
  --cluster manga-assist-prod \
  --service manga-assist-orchestrator \
  --desired-count 40

# Update SSM parameter (config source for ECS tasks)
aws ssm put-parameter \
  --name "/manga-assist/prod/bedrock-pool-size" \
  --value "25" \
  --type String \
  --overwrite

# Force new deployment to pick up the parameter
aws ecs update-service \
  --cluster manga-assist-prod \
  --service manga-assist-orchestrator \
  --force-new-deployment

aws ssm put-parameter \
  --name "/manga-assist/prod/cache-similarity-threshold" \
  --value "0.85" \
  --type String \
  --overwrite

graph TB
    subgraph "Capacity Planning"
        CP1["Model connection pool sizing<br/>for peak, not average"]
        CP2["Calculate: peak_rps x<br/>bedrock_miss_rate x<br/>avg_latency / 1000 =<br/>connections needed"]
        CP3["Add 50% headroom to<br/>calculated pool size"]
    end

    subgraph "Auto-Scaling"
        AS1["Scale on connection pool<br/>utilization, not just CPU"]
        AS2["Target: pool utilization<br/>< 70% triggers scale-out"]
        AS3["Pre-scale for known events<br/>(publisher releases, sales)"]
    end

    subgraph "Circuit Breaking"
        CB1["Bedrock circuit breaker<br/>with Haiku fallback"]
        CB2["If Sonnet pool exhausted,<br/>route to Haiku pool<br/>(separate pool, faster)"]
        CB3["Queue with timeout<br/>+ graceful degradation"]
    end

    CP1 --> CP2 --> CP3
    AS1 --> AS2 --> AS3
    CB1 --> CB2 --> CB3

    style CP1 fill:#2ecc71,color:#000
    style AS1 fill:#3498db,color:#fff
    style CB1 fill:#9b59b6,color:#fff

graph TB
    A["SRE reviews daily<br/>latency trend dashboard"]
    B["p50 latency increased<br/>25ms since Tuesday deploy"]
    C{"Correlate with deployment<br/>timeline"}
    D["Tuesday: DynamoDB VPC<br/>gateway endpoint deployed"]
    E{"Check per-segment<br/>latency trends"}
    F["dynamo-session segment:<br/>p50 dropped from 12ms to 6ms<br/>(good — VPC endpoint working)"]
    G["vector-search segment:<br/>p50 increased from 60ms to 85ms<br/>(bad — 25ms regression!)"]
    H{"Why would DynamoDB<br/>endpoint affect OpenSearch?"}
    I["NAT gateway was handling<br/>both DynamoDB and OpenSearch.<br/>DynamoDB traffic moved off NAT,<br/>but NAT config was changed<br/>during migration, affecting<br/>OpenSearch routing."]

    A --> B --> C --> D --> E
    E --> F
    E --> G
    G --> H --> I

    style B fill:#f39c12,color:#000
    style G fill:#e74c3c,color:#fff
    style I fill:#e74c3c,color:#fff

# Compare current route table with previous version
aws ec2 describe-route-tables \
  --route-table-ids rtb-xxxxx \
  --query 'RouteTables[].Routes[]' \
  --output table

aws ec2 replace-route \
  --route-table-id rtb-xxxxx \
  --destination-cidr-block 0.0.0.0/0 \
  --nat-gateway-id nat-aaaa1111  # original NAT in ap-northeast-1a

graph TB
    subgraph "Migration Safety"
        M1["Deploy VPC endpoints<br/>one at a time with<br/>24-hour soak between"]
        M2["Run VPCEndpointHealthChecker<br/>after every infra change"]
        M3["Terraform plan review:<br/>check for unintended<br/>route table changes"]
    end

    subgraph "Observability"
        O1["Per-segment latency<br/>trend alerting<br/>(not just p95 breach)"]
        O2["Anomaly detection on<br/>p50 latency per segment"]
        O3["NAT gateway traffic<br/>baseline comparison<br/>(detect unexpected routing)"]
    end

    subgraph "Rollback"
        R1["Terraform state snapshots<br/>before each migration step"]
        R2["Automated rollback trigger:<br/>if any segment p50 increases<br/>> 10ms post-deploy"]
        R3["Canary deployment for<br/>infrastructure changes"]
    end

    M1 --> M2 --> M3
    O1 --> O2 --> O3
    R1 --> R2 --> R3

    style M1 fill:#2ecc71,color:#000
    style O1 fill:#3498db,color:#fff
    style R1 fill:#9b59b6,color:#fff

graph TB
    A["CloudWatch Alarm:<br/>p95 latency increased<br/>from 850ms to 1,350ms"]
    B["Check recent deployments:<br/>Guardrails policy update<br/>deployed 2 hours ago"]
    C{"X-Ray trace analysis<br/>for high-latency requests"}
    D["guardrails-check segment:<br/>p95 = 480ms<br/>(was 40ms before update)"]
    E{"Drill into guardrails<br/>subsegment metadata"}
    F["New policy has 12 content<br/>filters (was 4).<br/>Each filter runs sequentially.<br/>Long responses (>500 tokens)<br/>take 400-500ms to evaluate."]

    A --> B --> C --> D --> E --> F

    style A fill:#e74c3c,color:#fff
    style D fill:#e74c3c,color:#fff
    style F fill:#e74c3c,color:#fff

Old: 4 filters x 10ms/filter = 40ms
New: 12 filters x 10ms/filter = 120ms (short responses)
New: 12 filters x 40ms/filter = 480ms (long responses, 500+ tokens)

# BEFORE: Synchronous guardrails (blocks response delivery)
response = await bedrock_invoke(prompt)
guardrails_result = await guardrails_check(response)  # 480ms blocking!
if guardrails_result.is_safe:
    await stream_to_client(response)

# AFTER: Async guardrails (stream immediately, check in background)
response = await bedrock_invoke(prompt)
await stream_to_client(response)  # Stream starts immediately

# Check guardrails asynchronously
asyncio.create_task(
    guardrails_check_and_remediate(session_id, response)
)

async def guardrails_check_and_remediate(session_id, response):
    result = await guardrails_check(response)
    if not result.is_safe:
        # Send a follow-up message to the user
        await send_websocket_message(
            session_id,
            "I need to revise my previous response. Let me provide "
            "updated information..."
        )
        # Log for review
        logger.warning("Guardrails flagged response post-stream: %s",
                       result.violations)
        # Emit metric for monitoring
        emit_metric("GuardrailsPostStreamViolation", 1)

async def chunked_guardrails(stream):
    buffer = ""
    for chunk in stream:
        buffer += chunk
        yield chunk  # Stream to client immediately

        if len(buffer) > 100:  # Evaluate every 100 tokens
            result = await guardrails_check(buffer)
            if not result.is_safe:
                yield "[Content revised for safety]"
                break
            buffer = ""  # Reset for next chunk

graph TB
    subgraph "Policy Management"
        PM1["Load test every Guardrails<br/>policy change in staging"]
        PM2["Benchmark: measure p95<br/>evaluation time at<br/>100, 300, 500, 1000 tokens"]
        PM3["Set latency budget<br/>for Guardrails: max 50ms"]
    end

    subgraph "Architecture"
        AR1["Default: async Guardrails<br/>(stream first, check later)"]
        AR2["Sync only for high-risk<br/>intents (order placement,<br/>payment info)"]
        AR3["Chunked evaluation<br/>for streaming responses"]
    end

    subgraph "Monitoring"
        MO1["Alert on Guardrails<br/>evaluation time > 100ms"]
        MO2["Track filter count<br/>and complexity metrics"]
        MO3["A/B test: sync vs async<br/>Guardrails impact on UX"]
    end

    PM1 --> PM2 --> PM3
    AR1 --> AR2 --> AR3
    MO1 --> MO2 --> MO3

    style PM1 fill:#2ecc71,color:#000
    style AR1 fill:#3498db,color:#fff
    style MO1 fill:#9b59b6,color:#fff

graph TB
    A["Weekly latency review:<br/>p95 trending up 70ms<br/>over 3 weeks"]
    B["ECS scaled from 8 to 12<br/>tasks — latency unchanged"]
    C{"X-Ray trace comparison:<br/>this week vs 3 weeks ago"}
    D["bedrock-invoke: unchanged (800ms)<br/>vector-search: unchanged (85ms)<br/>cache-lookup: unchanged (2ms)"]
    E["cache-write: 3ms -> 18ms<br/>(6x increase!)"]
    F{"Investigate cache-write<br/>subsegment"}
    G["Redis SET operation for<br/>caching responses now takes<br/>15-20ms instead of 2-4ms"]
    H{"Check Redis metrics"}
    I["Redis memory usage: 92%<br/>Eviction rate: 1,200/min<br/>Connected clients: 580<br/>— Redis under pressure"]
    J{"Why is cache-write<br/>on the critical path?"}
    K["Code review reveals:<br/>cache write is SYNCHRONOUS<br/>— await redis.set() is called<br/>BEFORE response is returned"]

    A --> B --> C --> D --> E --> F --> G --> H --> I --> J --> K

    style A fill:#f39c12,color:#000
    style E fill:#e74c3c,color:#fff
    style I fill:#e74c3c,color:#fff
    style K fill:#e74c3c,color:#fff

# The problematic code path
async def handle_query(query):
    response = await bedrock_invoke(prompt)
    await redis.set(cache_key, response, ex=3600)  # BLOCKS response!
    return response  # Response delayed by cache write

# FIXED: Async cache write (does not block response)
async def handle_query(query):
    response = await bedrock_invoke(prompt)

    # Fire-and-forget: schedule cache write, return immediately
    asyncio.create_task(
        safe_cache_write(cache_key, response)
    )

    return response  # Response returns immediately

async def safe_cache_write(key, value):
    """Cache write with error handling (never fails the request)."""
    try:
        await redis.set(key, value, ex=3600)
    except Exception as e:
        logger.warning("Cache write failed (non-critical): %s", e)
        emit_metric("CacheWriteFailure", 1)

aws ssm put-parameter \
  --name "/manga-assist/prod/cache-ttl-seconds" \
  --value "1800" \
  --type String \
  --overwrite

aws elasticache modify-replication-group \
  --replication-group-id manga-assist-cache \
  --cache-node-type cache.r6g.xlarge \
  --apply-immediately

# Emit cache write timing as a separate metric
async def timed_cache_write(key, value):
    start = time.monotonic()
    try:
        await redis.set(key, value, ex=1800)
        elapsed_ms = (time.monotonic() - start) * 1000
        emit_metric("CacheWriteLatency", elapsed_ms)
        if elapsed_ms > 10:
            logger.warning("Slow cache write: %.1fms for key %s", elapsed_ms, key)
    except Exception as e:
        logger.warning("Cache write failed: %s", e)
        emit_metric("CacheWriteFailure", 1)

graph TB
    subgraph "Architecture Principle"
        AP1["ALL cache writes must<br/>be async/fire-and-forget"]
        AP2["Cache is an optimization,<br/>not a dependency"]
        AP3["Code review checklist:<br/>no await on cache SET<br/>in request hot path"]
    end

    subgraph "Capacity Planning"
        CP1["Redis memory alarm<br/>at 75% (not 90%)"]
        CP2["Auto-scale Redis based<br/>on memory utilization"]
        CP3["Project cache growth:<br/>new_queries/day x avg_size<br/>x TTL = memory needed"]
    end

    subgraph "Monitoring"
        MO1["Track cache write latency<br/>as a separate metric"]
        MO2["Alert if cache write p95<br/>> 5ms (early warning)"]
        MO3["Dashboard: cache memory<br/>utilization trend over 30 days"]
    end

    AP1 --> AP2 --> AP3
    CP1 --> CP2 --> CP3
    MO1 --> MO2 --> MO3

    style AP1 fill:#2ecc71,color:#000
    style CP1 fill:#3498db,color:#fff
    style MO1 fill:#9b59b6,color:#fff

graph TB
    START["p95 Latency > 2000ms"]
    Q1{"X-Ray: Which segment<br/>is slowest?"}

    Q1 -->|"bedrock-invoke"| BQ1{"Bedrock throttling?<br/>ThrottlingException > 0?"}
    BQ1 -->|"Yes"| BR1["Scale down concurrency<br/>or request quota increase"]
    BQ1 -->|"No"| BQ2{"Network transit<br/>> 50ms?"}
    BQ2 -->|"Yes"| BR2["Check VPC endpoint<br/>+ DNS resolution<br/>(Scenario 1)"]
    BQ2 -->|"No"| BQ3{"Large prompt<br/>> 4000 tokens?"}
    BQ3 -->|"Yes"| BR3["Optimize prompt<br/>(Skill 4.1.1)"]
    BQ3 -->|"No"| BR4["Check Bedrock<br/>service health dashboard"]

    Q1 -->|"vector-search"| OQ1{"Query has filters?"}
    OQ1 -->|"No"| OR1["Add pre-filters<br/>(category, language, stock)"]
    OQ1 -->|"Yes"| OQ2{"Returning all fields?"}
    OQ2 -->|"Yes"| OR2["Add _source selection<br/>(return only 6 fields)"]
    OQ2 -->|"No"| OR3["Check OpenSearch OCU<br/>scaling + cold start"]

    Q1 -->|"cache-write/session-update"| CQ1{"Write is synchronous?"}
    CQ1 -->|"Yes"| CR1["Make async<br/>(Scenario 5)"]
    CQ1 -->|"No"| CQ2{"Redis memory > 80%?"}
    CQ2 -->|"Yes"| CR2["Scale Redis<br/>or reduce TTL"]
    CQ2 -->|"No"| CR3["Check network path<br/>to Redis endpoint"]

    Q1 -->|"guardrails-check"| GQ1{"Evaluation time<br/>> 100ms?"}
    GQ1 -->|"Yes"| GR1["Move to async<br/>(Scenario 4)"]
    GQ1 -->|"No"| GR2["Check filter count<br/>and response size"]

    Q1 -->|"All segments normal<br/>but total is high"| AQ1{"Connection pool<br/>utilization > 80%?"}
    AQ1 -->|"Yes"| AR1["Pool exhaustion<br/>(Scenario 2)"]
    AQ1 -->|"No"| AQ2{"DNS resolution<br/>> 10ms?"}
    AQ2 -->|"Yes"| AR2["DNS issue<br/>(Scenario 1)"]
    AQ2 -->|"No"| AR3["Check for new<br/>inter-service hops<br/>or cold ECS tasks"]

    style START fill:#e74c3c,color:#fff
    style BR1 fill:#2ecc71,color:#000
    style BR2 fill:#2ecc71,color:#000
    style BR3 fill:#2ecc71,color:#000
    style BR4 fill:#f39c12,color:#000
    style OR1 fill:#2ecc71,color:#000
    style OR2 fill:#2ecc71,color:#000
    style OR3 fill:#f39c12,color:#000
    style CR1 fill:#2ecc71,color:#000
    style CR2 fill:#2ecc71,color:#000
    style CR3 fill:#f39c12,color:#000
    style GR1 fill:#2ecc71,color:#000
    style GR2 fill:#f39c12,color:#000
    style AR1 fill:#2ecc71,color:#000
    style AR2 fill:#2ecc71,color:#000
    style AR3 fill:#f39c12,color:#000