curl http://localhost:8080/health

{
  "status": "healthy",
  "timestamp": "2024-01-21T12:00:00Z",
  "database": "connected",
  "redis": "connected"
}

curl http://localhost:8080/ready

{
  "status": "ready",
  "checks": {
    "sandbox_manager": true,
    "redis": true
  }
}

readinessProbe:
  httpGet:
    path: /ready
    port: 8080
  initialDelaySeconds: 5
  periodSeconds: 10

curl http://localhost:8080/health/detailed

{
  "timestamp": "2024-01-21T12:00:00Z",
  "environment": "production",
  "version": "1.0.0",
  "status": "healthy",
  "checks": {
    "database": {
      "status": "healthy"
    },
    "redis": {
      "status": "healthy",
      "details": {
        "required": false,
        "connected": true
      }
    },
    "sandbox_manager": {
      "status": "healthy"
    }
  }
}

curl -H "X-Internal-Token: $INTERNAL_API_KEY" http://localhost:8080/metrics

# prometheus.yml
scrape_configs:
  - job_name: 'sandbox-controller'
    static_configs:
      - targets: ['sandbox-controller:8080']
    metrics_path: /metrics
    authorization:
      type: Bearer
      credentials: '<INTERNAL_API_KEY>'
    scrape_interval: 15s

# Sandbox creation success rate (5m window)
sum(rate(stateset_sandbox_sandbox_creations_total{status="success"}[5m]))
/
sum(rate(stateset_sandbox_sandbox_creations_total[5m]))

# P99 cold start latency
histogram_quantile(0.99,
  sum(rate(stateset_sandbox_sandbox_startup_duration_seconds_bucket[5m])) by (le)
)

# Warm pool hit rate
sum(rate(stateset_sandbox_warm_pool_hits_total{result="hit"}[5m]))
/
sum(rate(stateset_sandbox_warm_pool_hits_total[5m]))

# Error rate by operation
sum(rate(stateset_sandbox_operation_errors_total[5m])) by (operation)

# Active sandboxes over time
stateset_sandbox_active_sandboxes

# alerts.yml
groups:
  - name: sandbox-controller
    rules:
      # High error rate
      - alert: HighErrorRate
        expr: |
          sum(rate(stateset_sandbox_operation_errors_total[5m]))
          / sum(rate(stateset_sandbox_http_requests_total[5m])) > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High error rate detected"
          description: "Error rate is {{ $value | humanizePercentage }}"

      # Controller down
      - alert: ControllerDown
        expr: up{job="sandbox-controller"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Sandbox controller is down"

      # Database connection issues
      - alert: DatabaseUnhealthy
        expr: |
          stateset_sandbox_background_task_failures_total{task_name="database_health"} > 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Database connection issues"

      # Warm pool exhausted
      - alert: WarmPoolExhausted
        expr: |
          rate(stateset_sandbox_warm_pool_hits_total{result="miss"}[5m])
          / rate(stateset_sandbox_warm_pool_hits_total[5m]) > 0.5
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Warm pool miss rate too high"
          description: "Consider increasing WARM_POOL_SIZE"

      # High latency
      - alert: HighLatency
        expr: |
          histogram_quantile(0.99,
            sum(rate(stateset_sandbox_http_request_duration_seconds_bucket[5m])) by (le)
          ) > 5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "P99 latency exceeds 5 seconds"

      # Sandbox creation failures
      - alert: SandboxCreationFailures
        expr: |
          rate(stateset_sandbox_sandbox_creations_total{status="failure"}[5m]) > 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Sandbox creation failures detected"

{
  "level": "info",
  "time": "2024-01-21T12:00:00.000Z",
  "msg": "Sandbox created",
  "sandbox_id": "sb-abc123",
  "org_id": "org-xyz",
  "duration_ms": 1234
}

# promtail config
scrape_configs:
  - job_name: sandbox-controller
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app]
        regex: sandbox-controller
        action: keep
    pipeline_stages:
      - json:
          expressions:
            level: level
            msg: msg
            sandbox_id: sandbox_id

# Errors in last hour
{app="sandbox-controller"} |= "error" | json | level="error"

# Specific sandbox logs
{app="sandbox-controller"} | json | sandbox_id="sb-abc123"

# Slow requests
{app="sandbox-controller"} | json | duration_ms > 5000

apiVersion: apps/v1
kind: Deployment
spec:
  replicas: 3
  template:
    spec:
      containers:
        - name: controller
          resources:
            requests:
              cpu: "500m"
              memory: "512Mi"
            limits:
              cpu: "2"
              memory: "2Gi"

# Backup
pg_dump "$DATABASE_URL" > backup.sql

# Restore
psql "$DATABASE_URL" < backup.sql

# Revoke compromised key
curl -X DELETE -H "Authorization: Bearer $ADMIN_KEY" \
  http://localhost:8080/api/admin/api-keys/<key-id>

SELECT * FROM audit_logs
WHERE action = 'sandbox.create'
AND created_at > NOW() - INTERVAL '24 hours'
ORDER BY created_at DESC;