{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "Prometheus-compatible datasource receiving OTLP metrics from awa workers",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "annotations": { "list": [] },
  "editable": true,
  "graphTooltip": 1,
  "links": [],
  "templating": {
    "list": [
      {
        "name": "queue",
        "label": "Queue",
        "type": "query",
        "query": "label_values(awa_job_in_flight, awa_job_queue)",
        "refresh": 2,
        "multi": true,
        "includeAll": true,
        "current": {},
        "datasource": { "uid": "${DS_PROMETHEUS}" }
      },
      {
        "name": "awa_ui_url",
        "label": "awa-ui base URL",
        "description": "Base URL of the awa-ui admin site. Panels drill-link to awa-ui for transactional views (pause, retry, drain) this dashboard intentionally doesn't duplicate.",
        "type": "textbox",
        "query": "http://localhost:3000",
        "current": { "value": "http://localhost:3000", "text": "http://localhost:3000" },
        "hide": 0
      }
    ]
  },
  "panels": [
    {
      "id": 100,
      "type": "stat",
      "title": "Error rate",
      "description": "failed / (completed + failed) over the last 5 minutes.",
      "gridPos": { "h": 4, "w": 5, "x": 0, "y": 0 },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "noValue": "0%",
          "thresholds": { "mode": "absolute", "steps": [
            { "color": "green",  "value": null },
            { "color": "orange", "value": 0.01 },
            { "color": "red",    "value": 0.05 }
          ]}
        }
      },
      "targets": [
        {
          "expr": "sum(rate(awa_job_failed_total{awa_job_queue=~\"$queue\"}[5m])) / clamp_min(sum(rate(awa_job_completed_total{awa_job_queue=~\"$queue\"}[5m])) + sum(rate(awa_job_failed_total{awa_job_queue=~\"$queue\"}[5m])), 1e-9)",
          "legendFormat": "error rate"
        }
      ],
      "datasource": { "uid": "${DS_PROMETHEUS}" }
    },
    {
      "id": 101,
      "type": "stat",
      "title": "In-flight",
      "description": "Total currently-executing jobs across all selected queues.",
      "gridPos": { "h": 4, "w": 5, "x": 5, "y": 0 },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }]},
          "links": [{ "title": "Open /runtime in awa-ui", "url": "${awa_ui_url}/runtime", "targetBlank": true }]
        }
      },
      "targets": [
        { "expr": "sum(awa_job_in_flight{awa_job_queue=~\"$queue\"})", "legendFormat": "in flight" }
      ],
      "datasource": { "uid": "${DS_PROMETHEUS}" }
    },
    {
      "id": 102,
      "type": "stat",
      "title": "Throughput (1m)",
      "description": "Completions per second averaged over the last minute.",
      "gridPos": { "h": 4, "w": 5, "x": 10, "y": 0 },
      "fieldConfig": {
        "defaults": {
          "unit": "ops",
          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }]}
        }
      },
      "targets": [
        { "expr": "sum(rate(awa_job_completed_total{awa_job_queue=~\"$queue\"}[1m]))", "legendFormat": "throughput" }
      ],
      "datasource": { "uid": "${DS_PROMETHEUS}" }
    },
    {
      "id": 103,
      "type": "stat",
      "title": "Rescues (5m)",
      "description": "Jobs rescued by heartbeat, deadline, or callback-timeout in the last 5 minutes. Non-zero means workers are losing grip of jobs.",
      "gridPos": { "h": 4, "w": 5, "x": 15, "y": 0 },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "noValue": "0",
          "thresholds": { "mode": "absolute", "steps": [
            { "color": "green",  "value": null },
            { "color": "orange", "value": 1 },
            { "color": "red",    "value": 10 }
          ]}
        }
      },
      "targets": [
        { "expr": "sum(increase(awa_maintenance_rescues_total[5m]))", "legendFormat": "rescues" }
      ],
      "datasource": { "uid": "${DS_PROMETHEUS}" }
    },
    {
      "id": 104,
      "type": "stat",
      "title": "Active runtimes",
      "description": "Distinct runtime instances that emitted the in-flight gauge in the last minute. Falls to 0 if the whole fleet is down.",
      "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "thresholds": { "mode": "absolute", "steps": [
            { "color": "red",   "value": null },
            { "color": "green", "value": 1 }
          ]}
        }
      },
      "targets": [
        { "expr": "count(count by (service_name, instance) (awa_job_in_flight))", "legendFormat": "runtimes" }
      ],
      "datasource": { "uid": "${DS_PROMETHEUS}" }
    },
    {
      "id": 110,
      "type": "timeseries",
      "title": "Queue Lag",
      "description": "Age of the oldest available job per queue. The #1 operator signal — jumps when workers can't keep up. Display name lifted in via the awa_queue_info join.",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10 },
          "thresholds": { "mode": "absolute", "steps": [
            { "color": "green",  "value": null },
            { "color": "yellow", "value": 30 },
            { "color": "red",    "value": 120 }
          ]},
          "links": [
            { "title": "Open queue in awa-ui", "url": "${awa_ui_url}/queues/${__field.labels.awa_job_queue}", "targetBlank": true }
          ]
        }
      },
      "targets": [
        {
          "expr": "(awa_queue_lag_seconds{awa_job_queue=~\"$queue\"} * on(awa_job_queue) group_left(awa_queue_display_name) awa_queue_info) or label_replace(awa_queue_lag_seconds{awa_job_queue=~\"$queue\"}, \"awa_queue_display_name\", \"$1\", \"awa_job_queue\", \"(.*)\")",
          "legendFormat": "{{awa_queue_display_name}} ({{awa_job_queue}})"
        }
      ],
      "datasource": { "uid": "${DS_PROMETHEUS}" }
    },
    {
      "id": 111,
      "type": "timeseries",
      "title": "Queue Depth",
      "description": "Stacked depth per queue and state. Backlog shape + execution pressure at a glance.",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "stacking": { "mode": "normal" } },
          "links": [
            { "title": "Open queue in awa-ui", "url": "${awa_ui_url}/queues/${__field.labels.awa_job_queue}", "targetBlank": true }
          ]
        },
        "overrides": [
          { "matcher": { "id": "byRegexp", "options": "available" },        "properties": [{ "id": "color", "value": { "fixedColor": "green",  "mode": "fixed" } }] },
          { "matcher": { "id": "byRegexp", "options": "running" },          "properties": [{ "id": "color", "value": { "fixedColor": "blue",   "mode": "fixed" } }] },
          { "matcher": { "id": "byRegexp", "options": "failed" },           "properties": [{ "id": "color", "value": { "fixedColor": "red",    "mode": "fixed" } }] },
          { "matcher": { "id": "byRegexp", "options": "retryable" },        "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] },
          { "matcher": { "id": "byRegexp", "options": "scheduled" },        "properties": [{ "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } }] },
          { "matcher": { "id": "byRegexp", "options": "waiting_external" }, "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] }
        ]
      },
      "targets": [
        {
          "expr": "awa_queue_depth{awa_job_queue=~\"$queue\"}",
          "legendFormat": "{{awa_job_queue}} {{awa_job_state}}"
        }
      ],
      "datasource": { "uid": "${DS_PROMETHEUS}" }
    },
    {
      "collapsed": true,
      "id": 200,
      "type": "row",
      "title": "Latency",
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 },
      "panels": [
        {
          "id": 201,
          "type": "timeseries",
          "title": "Job Wait Time (p50 / p95 / p99)",
          "description": "Time from job creation to claim. Rising wait time with steady throughput = under-provisioned workers. Rising with steady workers = producer burst.",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
          "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0 } } },
          "targets": [
            { "expr": "histogram_quantile(0.50, sum by (le, awa_job_queue) (rate(awa_job_wait_duration_seconds_bucket{awa_job_queue=~\"$queue\"}[$__rate_interval])))", "legendFormat": "p50 {{awa_job_queue}}" },
            { "expr": "histogram_quantile(0.95, sum by (le, awa_job_queue) (rate(awa_job_wait_duration_seconds_bucket{awa_job_queue=~\"$queue\"}[$__rate_interval])))", "legendFormat": "p95 {{awa_job_queue}}" },
            { "expr": "histogram_quantile(0.99, sum by (le, awa_job_queue) (rate(awa_job_wait_duration_seconds_bucket{awa_job_queue=~\"$queue\"}[$__rate_interval])))", "legendFormat": "p99 {{awa_job_queue}}" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 202,
          "type": "timeseries",
          "title": "Job Duration (p50 / p95 / p99)",
          "description": "Execution time per queue. A regression in duration usually precedes a backlog alert.",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
          "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0 } } },
          "targets": [
            { "expr": "histogram_quantile(0.50, sum by (le, awa_job_queue) (rate(awa_job_duration_seconds_bucket{awa_job_queue=~\"$queue\"}[$__rate_interval])))", "legendFormat": "p50 {{awa_job_queue}}" },
            { "expr": "histogram_quantile(0.95, sum by (le, awa_job_queue) (rate(awa_job_duration_seconds_bucket{awa_job_queue=~\"$queue\"}[$__rate_interval])))", "legendFormat": "p95 {{awa_job_queue}}" },
            { "expr": "histogram_quantile(0.99, sum by (le, awa_job_queue) (rate(awa_job_duration_seconds_bucket{awa_job_queue=~\"$queue\"}[$__rate_interval])))", "legendFormat": "p99 {{awa_job_queue}}" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        }
      ]
    },
    {
      "collapsed": true,
      "id": 300,
      "type": "row",
      "title": "Throughput",
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 },
      "panels": [
        {
          "id": 301,
          "type": "timeseries",
          "title": "Completions / failures / retries / cancellations",
          "description": "Per-result rate across the selected queues.",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 },
          "fieldConfig": {
            "defaults": { "unit": "ops", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "stacking": { "mode": "normal" } } },
            "overrides": [
              { "matcher": { "id": "byRegexp", "options": "completed" }, "properties": [{ "id": "color", "value": { "fixedColor": "green",  "mode": "fixed" } }] },
              { "matcher": { "id": "byRegexp", "options": "failed" },    "properties": [{ "id": "color", "value": { "fixedColor": "red",    "mode": "fixed" } }] },
              { "matcher": { "id": "byRegexp", "options": "retried" },   "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] },
              { "matcher": { "id": "byRegexp", "options": "cancelled" }, "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] }
            ]
          },
          "targets": [
            { "expr": "sum(rate(awa_job_completed_total{awa_job_queue=~\"$queue\"}[$__rate_interval]))", "legendFormat": "completed" },
            { "expr": "sum(rate(awa_job_failed_total{awa_job_queue=~\"$queue\"}[$__rate_interval]))",    "legendFormat": "failed" },
            { "expr": "sum(rate(awa_job_retried_total{awa_job_queue=~\"$queue\"}[$__rate_interval]))",   "legendFormat": "retried" },
            { "expr": "sum(rate(awa_job_cancelled_total{awa_job_queue=~\"$queue\"}[$__rate_interval]))", "legendFormat": "cancelled" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 302,
          "type": "timeseries",
          "title": "Throughput by Queue (with display names)",
          "description": "Completed jobs/s per queue with the declared display name lifted in via awa_queue_info. Falls back to raw queue name when no descriptor is declared.",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 },
          "fieldConfig": {
            "defaults": {
              "unit": "ops",
              "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10 },
              "links": [
                { "title": "Open queue in awa-ui", "url": "${awa_ui_url}/queues/${__field.labels.awa_job_queue}", "targetBlank": true }
              ]
            }
          },
          "targets": [
            {
              "expr": "sum by (awa_job_queue, awa_queue_display_name) (\n  rate(awa_job_completed_total{awa_job_queue=~\"$queue\"}[$__rate_interval])\n  * on(awa_job_queue) group_left(awa_queue_display_name) awa_queue_info\n) or label_replace(\n  sum by (awa_job_queue) (rate(awa_job_completed_total{awa_job_queue=~\"$queue\"}[$__rate_interval])),\n  \"awa_queue_display_name\", \"$1\", \"awa_job_queue\", \"(.*)\"\n)",
              "legendFormat": "{{awa_queue_display_name}} ({{awa_job_queue}})"
            }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 303,
          "type": "timeseries",
          "title": "Top 10 Kinds by Throughput",
          "description": "Completed jobs/s broken down by kind, capped at the 10 busiest. Click through to /jobs?q=kind:... in awa-ui.",
          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 22 },
          "fieldConfig": {
            "defaults": {
              "unit": "ops",
              "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 80, "stacking": { "mode": "normal" } },
              "links": [
                { "title": "Open /jobs (kind filter) in awa-ui", "url": "${awa_ui_url}/jobs?q=kind:${__field.labels.awa_job_kind}", "targetBlank": true }
              ]
            }
          },
          "targets": [
            { "expr": "topk(10, sum by (awa_job_kind) (rate(awa_job_completed_total{awa_job_queue=~\"$queue\"}[$__rate_interval])))", "legendFormat": "{{awa_job_kind}}" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        }
      ]
    },
    {
      "collapsed": true,
      "id": 400,
      "type": "row",
      "title": "Worker internals",
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
      "panels": [
        {
          "id": 401,
          "type": "timeseries",
          "title": "Claim Latency (p50 / p95)",
          "description": "Postgres dequeue query time per queue.",
          "gridPos": { "h": 8, "w": 8, "x": 0, "y": 15 },
          "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0 } } },
          "targets": [
            { "expr": "histogram_quantile(0.50, sum by (le, awa_job_queue) (rate(awa_dispatch_claim_duration_seconds_bucket{awa_job_queue=~\"$queue\"}[$__rate_interval])))", "legendFormat": "p50 {{awa_job_queue}}" },
            { "expr": "histogram_quantile(0.95, sum by (le, awa_job_queue) (rate(awa_dispatch_claim_duration_seconds_bucket{awa_job_queue=~\"$queue\"}[$__rate_interval])))", "legendFormat": "p95 {{awa_job_queue}}" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 402,
          "type": "timeseries",
          "title": "Claim Batch Size",
          "description": "Average jobs claimed per dispatch poll. High values indicate the claim loop is keeping up; 1s mean bursty traffic.",
          "gridPos": { "h": 8, "w": 8, "x": 8, "y": 15 },
          "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10 } } },
          "targets": [
            {
              "expr": "sum by (awa_job_queue) (rate(awa_dispatch_claim_batch_size_sum{awa_job_queue=~\"$queue\"}[$__rate_interval])) / clamp_min(sum by (awa_job_queue) (rate(awa_dispatch_claim_batch_size_count{awa_job_queue=~\"$queue\"}[$__rate_interval])), 1e-9)",
              "legendFormat": "avg {{awa_job_queue}}"
            }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 403,
          "type": "timeseries",
          "title": "Maintenance Rescues",
          "description": "Rate of rescued jobs by kind.",
          "gridPos": { "h": 8, "w": 8, "x": 16, "y": 15 },
          "fieldConfig": { "defaults": { "unit": "ops", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "stacking": { "mode": "normal" } } } },
          "targets": [
            { "expr": "sum by (awa_rescue_kind) (rate(awa_maintenance_rescues_total[$__rate_interval]))", "legendFormat": "{{awa_rescue_kind}}" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 404,
          "type": "timeseries",
          "title": "Completion Flush Performance",
          "description": "Time per batched completion flush (p50/p95).",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 },
          "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0 } } },
          "targets": [
            { "expr": "histogram_quantile(0.50, sum by (le) (rate(awa_completion_flush_duration_seconds_bucket[$__rate_interval])))", "legendFormat": "p50 flush" },
            { "expr": "histogram_quantile(0.95, sum by (le) (rate(awa_completion_flush_duration_seconds_bucket[$__rate_interval])))", "legendFormat": "p95 flush" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 405,
          "type": "timeseries",
          "title": "Promotion Throughput",
          "description": "Rate of scheduled/retryable jobs promoted to available by the maintenance leader.",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 },
          "fieldConfig": { "defaults": { "unit": "ops", "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 60, "stacking": { "mode": "normal" } } } },
          "targets": [
            { "expr": "sum by (awa_job_state) (rate(awa_maintenance_promote_batches_total[$__rate_interval]))", "legendFormat": "{{awa_job_state}} promoted/s" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        }
      ]
    },
    {
      "collapsed": true,
      "id": 500,
      "type": "row",
      "title": "Descriptor catalogs (summary; full tables in awa-ui)",
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 },
      "panels": [
        {
          "id": 501,
          "type": "table",
          "title": "Declared queues · joins to runtime metrics via awa_queue_info",
          "description": "awa_queue_info is always 1; it's a label-join target. Full queue table (with pause/drain actions) lives at /queues in awa-ui.",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
          "fieldConfig": {
            "defaults": { "custom": { "align": "left" } },
            "overrides": [
              { "matcher": { "id": "byName", "options": "awa_job_queue" },         "properties": [{ "id": "displayName", "value": "queue" }, { "id": "custom.width", "value": 180 }, { "id": "links", "value": [{ "title": "Open in awa-ui", "url": "${awa_ui_url}/queues/${__value.text}", "targetBlank": true }]}] },
              { "matcher": { "id": "byName", "options": "awa_queue_display_name" }, "properties": [{ "id": "displayName", "value": "display name" }, { "id": "custom.width", "value": 200 }] },
              { "matcher": { "id": "byName", "options": "awa_queue_owner" },        "properties": [{ "id": "displayName", "value": "owner" }, { "id": "custom.width", "value": 160 }] },
              { "matcher": { "id": "byName", "options": "awa_queue_tags" },         "properties": [{ "id": "displayName", "value": "tags" }] },
              { "matcher": { "id": "byName", "options": "awa_queue_docs_url" },     "properties": [{ "id": "displayName", "value": "docs" }] },
              { "matcher": { "id": "byName", "options": "Value" },                   "properties": [{ "id": "custom.hidden", "value": true }] }
            ]
          },
          "transformations": [
            { "id": "labelsToFields", "options": { "mode": "columns" } },
            { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true, "service_name": true, "awa_queue_description": true }, "indexByName": {} } }
          ],
          "targets": [
            { "expr": "awa_queue_info", "instant": true, "format": "table", "legendFormat": "" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 502,
          "type": "table",
          "title": "Declared job kinds · joins to runtime metrics via awa_job_kind_info",
          "description": "awa_job_kind_info is always 1; it's a label-join target for per-kind panels. Full kind table with per-kind metadata lives at /kinds in awa-ui.",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
          "fieldConfig": {
            "defaults": { "custom": { "align": "left" } },
            "overrides": [
              { "matcher": { "id": "byName", "options": "awa_job_kind" },              "properties": [{ "id": "displayName", "value": "kind" }, { "id": "custom.width", "value": 200 }, { "id": "links", "value": [{ "title": "Open in awa-ui", "url": "${awa_ui_url}/jobs?q=kind:${__value.text}", "targetBlank": true }]}] },
              { "matcher": { "id": "byName", "options": "awa_job_kind_display_name" }, "properties": [{ "id": "displayName", "value": "display name" }, { "id": "custom.width", "value": 200 }] },
              { "matcher": { "id": "byName", "options": "awa_job_kind_owner" },        "properties": [{ "id": "displayName", "value": "owner" }, { "id": "custom.width", "value": 160 }] },
              { "matcher": { "id": "byName", "options": "awa_job_kind_tags" },         "properties": [{ "id": "displayName", "value": "tags" }] },
              { "matcher": { "id": "byName", "options": "awa_job_kind_docs_url" },     "properties": [{ "id": "displayName", "value": "docs" }] },
              { "matcher": { "id": "byName", "options": "Value" },                     "properties": [{ "id": "custom.hidden", "value": true }] }
            ]
          },
          "transformations": [
            { "id": "labelsToFields", "options": { "mode": "columns" } },
            { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true, "service_name": true, "awa_job_kind_description": true }, "indexByName": {} } }
          ],
          "targets": [
            { "expr": "awa_job_kind_info", "instant": true, "format": "table", "legendFormat": "" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        }
      ]
    },
    {
      "collapsed": true,
      "id": 600,
      "type": "row",
      "title": "Storage transition",
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 },
      "panels": [
        {
          "id": 601,
          "type": "table",
          "title": "Current storage transition state",
          "description": "One-hot state/engine gauge exported by queue-storage-capable runtimes. Use this to distinguish steady-state canonical operation from prepared or mixed-transition rollout windows.",
          "gridPos": { "h": 7, "w": 10, "x": 0, "y": 25 },
          "fieldConfig": {
            "defaults": { "custom": { "align": "left" } },
            "overrides": [
              { "matcher": { "id": "byName", "options": "awa_storage_state" }, "properties": [{ "id": "displayName", "value": "state" }] },
              { "matcher": { "id": "byName", "options": "awa_storage_current_engine" }, "properties": [{ "id": "displayName", "value": "current engine" }] },
              { "matcher": { "id": "byName", "options": "awa_storage_active_engine" }, "properties": [{ "id": "displayName", "value": "active engine" }] },
              { "matcher": { "id": "byName", "options": "awa_storage_prepared_engine" }, "properties": [{ "id": "displayName", "value": "prepared engine" }] },
              { "matcher": { "id": "byName", "options": "Value" }, "properties": [{ "id": "custom.hidden", "value": true }] }
            ]
          },
          "transformations": [
            { "id": "labelsToFields", "options": { "mode": "columns" } },
            { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true, "service_name": true }, "indexByName": {} } }
          ],
          "targets": [
            { "expr": "awa_storage_state", "instant": true, "format": "table", "legendFormat": "" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 602,
          "type": "stat",
          "title": "Canonical live backlog",
          "description": "Jobs still live in canonical tables. Must reach zero before finalize is safe.",
          "gridPos": { "h": 7, "w": 4, "x": 10, "y": 25 },
          "fieldConfig": {
            "defaults": {
              "unit": "short",
              "thresholds": { "mode": "absolute", "steps": [
                { "color": "green", "value": null },
                { "color": "orange", "value": 1 }
              ]}
            }
          },
          "targets": [
            { "expr": "max(awa_storage_canonical_live_backlog)", "legendFormat": "backlog" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 603,
          "type": "stat",
          "title": "Enter mixed transition",
          "description": "1 means the fleet is ready to flip routing into queue storage. 0 means a blocker remains.",
          "gridPos": { "h": 7, "w": 5, "x": 14, "y": 25 },
          "fieldConfig": {
            "defaults": {
              "unit": "none",
              "mappings": [
                { "type": "value", "options": { "0": { "text": "Blocked" }, "1": { "text": "Ready" } } }
              ],
              "thresholds": { "mode": "absolute", "steps": [
                { "color": "red", "value": null },
                { "color": "green", "value": 1 }
              ]}
            }
          },
          "targets": [
            { "expr": "max(awa_storage_transition_ready{awa_storage_action=\"enter_mixed_transition\"})", "legendFormat": "enter-mixed-transition" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 604,
          "type": "stat",
          "title": "Finalize",
          "description": "1 means canonical backlog and runtime-capability gates are clear for finalization.",
          "gridPos": { "h": 7, "w": 5, "x": 19, "y": 25 },
          "fieldConfig": {
            "defaults": {
              "unit": "none",
              "mappings": [
                { "type": "value", "options": { "0": { "text": "Blocked" }, "1": { "text": "Ready" } } }
              ],
              "thresholds": { "mode": "absolute", "steps": [
                { "color": "red", "value": null },
                { "color": "green", "value": 1 }
              ]}
            }
          },
          "targets": [
            { "expr": "max(awa_storage_transition_ready{awa_storage_action=\"finalize\"})", "legendFormat": "finalize" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        },
        {
          "id": 605,
          "type": "timeseries",
          "title": "Live runtime capability counts",
          "description": "How many live runtimes currently report each storage capability. During mixed rollout you should see queue_storage targets before the routing flip, then canonical_drain_only fall to zero before finalize.",
          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 32 },
          "fieldConfig": {
            "defaults": {
              "unit": "short",
              "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 15 }
            }
          },
          "targets": [
            { "expr": "awa_storage_live_runtime_capability", "legendFormat": "{{awa_storage_capability}}" }
          ],
          "datasource": { "uid": "${DS_PROMETHEUS}" }
        }
      ]
    }
  ,
    {
      "id": 700,
      "type": "row",
      "title": "Ring rotation & prune (queue-storage)",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 41,
        "w": 24,
        "h": 1
      },
      "panels": [
        {
          "id": 701,
          "type": "timeseries",
          "title": "Rotation rate by ring + outcome",
          "description": "Ring rotation attempts/sec, split by ring (queue/lease/claim) and outcome. Healthy rings show non-zero `rotated` on a regular cadence; pinned rings dominate `skipped_busy`.",
          "gridPos": {
            "x": 0,
            "y": 42,
            "w": 12,
            "h": 8
          },
          "fieldConfig": {
            "defaults": {
              "custom": {
                "drawStyle": "line",
                "lineWidth": 2,
                "fillOpacity": 10
              },
              "unit": "ops"
            }
          },
          "targets": [
            {
              "expr": "sum by(awa_ring, awa_ring_outcome) (rate(awa_maintenance_rotate_attempts_total[1m]))",
              "legendFormat": "{{awa_ring}} \u00b7 {{awa_ring_outcome}}"
            }
          ],
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          }
        },
        {
          "id": 702,
          "type": "timeseries",
          "title": "Rotation skip blockers (queue ring)",
          "description": "When queue rotation is SkippedBusy, which queue/lease/claim partition family held it. These are bounded presence signals, not exact row counts.",
          "gridPos": {
            "x": 12,
            "y": 42,
            "w": 12,
            "h": 8
          },
          "fieldConfig": {
            "defaults": {
              "custom": {
                "drawStyle": "line",
                "lineWidth": 2,
                "fillOpacity": 10
              },
              "unit": "ops"
            }
          },
          "targets": [
            {
              "expr": "sum by(awa_ring_blocker) (rate(awa_maintenance_rotate_attempts_total{awa_ring=\"queue\", awa_ring_outcome=\"skipped_busy\"}[1m]))",
              "legendFormat": "{{awa_ring_blocker}}"
            }
          ],
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          }
        },
        {
          "id": 703,
          "type": "timeseries",
          "title": "Rotation skip blocker presence p95",
          "description": "p95 non-empty signal seen at the blocker side when rotation is SkippedBusy. Values are bounded to indicate presence; use queue depth and partition size panels for backlog magnitude.",
          "gridPos": {
            "x": 0,
            "y": 50,
            "w": 12,
            "h": 8
          },
          "fieldConfig": {
            "defaults": {
              "custom": {
                "drawStyle": "line",
                "lineWidth": 2,
                "fillOpacity": 10
              },
              "unit": "short"
            }
          },
          "targets": [
            {
              "expr": "histogram_quantile(0.95, sum by(le, awa_ring, awa_ring_blocker) (rate(awa_maintenance_rotate_skipped_rows_bucket[5m])))",
              "legendFormat": "{{awa_ring}} \u00b7 {{awa_ring_blocker}}"
            }
          ],
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          }
        },
        {
          "id": 704,
          "type": "timeseries",
          "title": "Prune rate by ring + outcome",
          "description": "Ring prune attempts/sec by outcome. `pruned` = destructive reclaim. `already_pruned` = idle tick avoided repeat DDL. `skipped_active` = target still live. `blocked` = lock contention.",
          "gridPos": {
            "x": 12,
            "y": 50,
            "w": 12,
            "h": 8
          },
          "fieldConfig": {
            "defaults": {
              "custom": {
                "drawStyle": "line",
                "lineWidth": 2,
                "fillOpacity": 10
              },
              "unit": "ops"
            }
          },
          "targets": [
            {
              "expr": "sum by(awa_ring, awa_ring_outcome) (rate(awa_maintenance_prune_attempts_total[1m]))",
              "legendFormat": "{{awa_ring}} \u00b7 {{awa_ring_outcome}}"
            }
          ],
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          }
        },
        {
          "id": 705,
          "type": "timeseries",
          "title": "Prune skip reasons (all rings)",
          "description": "When prune returns SkippedActive, which gate fired? `queue.active_leases` = leases on prior generation. `queue.pending_ready` = ready rows without matching done. `claim.open` = open receipt-plane claims. `lease.current`/`claim.current` = rotator race (benign if rare).",
          "gridPos": {
            "x": 0,
            "y": 58,
            "w": 12,
            "h": 8
          },
          "fieldConfig": {
            "defaults": {
              "custom": {
                "drawStyle": "line",
                "lineWidth": 2,
                "fillOpacity": 10
              },
              "unit": "ops"
            }
          },
          "targets": [
            {
              "expr": "sum by(awa_ring_reason) (rate(awa_maintenance_prune_attempts_total{awa_ring_outcome=\"skipped_active\"}[1m]))",
              "legendFormat": "{{awa_ring_reason}}"
            }
          ],
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          }
        },
        {
          "id": 706,
          "type": "timeseries",
          "title": "Rotations per minute (generation derivative)",
          "description": "Rate of ring generation advance. Headline 'is rotation working' signal \u2014 flat at zero with non-zero `skipped_busy` traffic above means the ring is pinned.",
          "gridPos": {
            "x": 12,
            "y": 58,
            "w": 12,
            "h": 8
          },
          "fieldConfig": {
            "defaults": {
              "custom": {
                "drawStyle": "line",
                "lineWidth": 2,
                "fillOpacity": 10
              },
              "unit": "opm"
            }
          },
          "targets": [
            {
              "expr": "60 * deriv(awa_ring_generation[1m])",
              "legendFormat": "{{awa_ring}}"
            }
          ],
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          }
        },
        {
          "id": 707,
          "type": "timeseries",
          "title": "Prune database phase latency p99",
          "description": "Successful destructive prune latency split by explicit child lock, TRUNCATE, and COMMIT. A high truncate phase with low lock time points to database storage/WAL latency rather than relation contention.",
          "gridPos": {
            "x": 0,
            "y": 66,
            "w": 24,
            "h": 8
          },
          "fieldConfig": {
            "defaults": {
              "custom": {
                "drawStyle": "line",
                "lineWidth": 2,
                "fillOpacity": 10
              },
              "unit": "s"
            }
          },
          "targets": [
            {
              "expr": "histogram_quantile(0.99, sum by(le, awa_ring, awa_maintenance_phase) (rate(awa_maintenance_prune_duration_seconds_bucket[5m])))",
              "legendFormat": "{{awa_ring}} · {{awa_maintenance_phase}}"
            }
          ],
          "datasource": {
            "uid": "${DS_PROMETHEUS}"
          }
        }
      ]
    }
  ],
  "schemaVersion": 39,
  "tags": ["awa", "job-queue"],
  "time": { "from": "now-5m", "to": "now" },
  "timepicker": {},
  "timezone": "",
  "title": "Awa Job Queue",
  "uid": "awa-job-queue",
  "version": 5,
  "refresh": "30s"
}