{
  "uid": "chat-delivery",
  "title": "chat — delivery service",
  "description": "Every number this deployment knows about itself. The counters that matter most are the ones for things the service survives quietly: dropped pushes, unenforced rate limits, reused KeyPackages.",
  "tags": [
    "chat"
  ],
  "timezone": "browser",
  "schemaVersion": 39,
  "version": 1,
  "refresh": "30s",
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "editable": true,
  "panels": [
    {
      "type": "row",
      "title": "Is it up",
      "gridPos": {
        "x": 0,
        "y": 0,
        "w": 24,
        "h": 1
      },
      "collapsed": false,
      "panels": []
    },
    {
      "type": "stat",
      "title": "Instances answering",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 0,
        "y": 1,
        "w": 4,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "count(up{job=\"chat-delivery\"} == 1)",
          "legendFormat": "up",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "description": "Targets Prometheus reached. Compare with the registry's own count beside it: they disagree when an instance died without unregistering."
    },
    {
      "type": "stat",
      "title": "Instances in the registry",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 4,
        "y": 1,
        "w": 4,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "max(chat_fleet_instances)",
          "legendFormat": "registered",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      }
    },
    {
      "type": "stat",
      "title": "Live sockets",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 8,
        "y": 1,
        "w": 4,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(chat_ws_sockets_open)",
          "legendFormat": "sockets",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "description": "Summed across the fleet: a socket is a file descriptor, so each instance only knows its own."
    },
    {
      "type": "stat",
      "title": "Since housekeeping last finished",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 12,
        "y": 1,
        "w": 5,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "time() - max(chat_maintenance_last_success_timestamp_seconds)",
          "legendFormat": "age",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "unit": "s"
        },
        "overrides": []
      },
      "description": "It runs every six hours on one instance of however many. Over a day is the alert."
    },
    {
      "type": "stat",
      "title": "Requests in flight",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 17,
        "y": 1,
        "w": 3,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(chat_http_requests_in_flight)",
          "legendFormat": "in flight",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      }
    },
    {
      "type": "stat",
      "title": "Bots served",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 20,
        "y": 1,
        "w": 4,
        "h": 4
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(chat_bots_serving)",
          "legendFormat": "bots",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      }
    },
    {
      "type": "row",
      "title": "Traffic",
      "gridPos": {
        "x": 0,
        "y": 5,
        "w": 24,
        "h": 1
      },
      "collapsed": false,
      "panels": []
    },
    {
      "type": "timeseries",
      "title": "Requests per second, by route",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 0,
        "y": 6,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (route) (rate(chat_http_requests_total[5m]))",
          "legendFormat": "{{route}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "unit": "reqps"
        },
        "overrides": []
      }
    },
    {
      "type": "timeseries",
      "title": "95th percentile, by route",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 12,
        "y": 6,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "histogram_quantile(0.95, sum by (le, route) (rate(chat_http_request_duration_seconds_bucket[5m])))",
          "legendFormat": "{{route}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "unit": "s"
        },
        "overrides": []
      },
      "description": "The label is the route template. It is never the path -- a group id in a label is an identifier kept for a year."
    },
    {
      "type": "timeseries",
      "title": "Failing requests",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 0,
        "y": 14,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(chat_http_requests_total{status=~\"5..\"}[5m])) / sum(rate(chat_http_requests_total[5m]))",
          "legendFormat": "5xx share",
          "refId": "A",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(chat_http_requests_total{status=\"429\"}[5m]))",
          "legendFormat": "429 per second",
          "refId": "B",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "unit": "percentunit"
        },
        "overrides": []
      }
    },
    {
      "type": "timeseries",
      "title": "Throttling",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 8,
        "y": 14,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (bucket) (rate(chat_rate_limited_total[5m]))",
          "legendFormat": "refused: {{bucket}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (bucket) (rate(chat_rate_bucket_faults_total[5m]))",
          "legendFormat": "UNENFORCED: {{bucket}}",
          "refId": "B",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "description": "The second series is the one that matters: the limiter fails open, so a bucket it cannot read is a limit that is not in force."
    },
    {
      "type": "timeseries",
      "title": "Logins and sessions",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 16,
        "y": 14,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (result) (rate(chat_auth_verifications_total[5m]))",
          "legendFormat": "verify: {{result}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (from) (rate(chat_sessions_resolved_total[5m]))",
          "legendFormat": "resolved from {{from}}",
          "refId": "B",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      }
    },
    {
      "type": "row",
      "title": "Delivery",
      "gridPos": {
        "x": 0,
        "y": 21,
        "w": 24,
        "h": 1
      },
      "collapsed": false,
      "panels": []
    },
    {
      "type": "timeseries",
      "title": "The group log",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 0,
        "y": 22,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (kind) (rate(chat_log_appends_total[5m]))",
          "legendFormat": "{{kind}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(chat_epoch_conflicts_total[5m]))",
          "legendFormat": "epoch conflicts",
          "refId": "B",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "description": "A few conflicts is two members writing at once and one rebasing. A sustained share of them is a group that cannot converge."
    },
    {
      "type": "timeseries",
      "title": "Live events",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 8,
        "y": 22,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(chat_ws_events_sent_total[5m]))",
          "legendFormat": "sent",
          "refId": "A",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (reason) (rate(chat_ws_events_dropped_total[5m]))",
          "legendFormat": "dropped: {{reason}}",
          "refId": "B",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "description": "Dropping is survivable -- clients reconcile against the log -- and invisible everywhere else."
    },
    {
      "type": "timeseries",
      "title": "Fanout between instances",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 16,
        "y": 22,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(chat_bus_envelopes_sent_total[5m]))",
          "legendFormat": "sent",
          "refId": "A",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(chat_bus_envelopes_received_total[5m]))",
          "legendFormat": "received",
          "refId": "B",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (stage) (rate(chat_bus_faults_total[5m]))",
          "legendFormat": "fault: {{stage}}",
          "refId": "C",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(chat_bus_reconnects_total[5m]))",
          "legendFormat": "reconnects",
          "refId": "D",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      }
    },
    {
      "type": "timeseries",
      "title": "Inbox",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 0,
        "y": 29,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (result) (rate(chat_inbox_deliveries_total[5m]))",
          "legendFormat": "{{result}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      }
    },
    {
      "type": "timeseries",
      "title": "KeyPackages",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 8,
        "y": 29,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (served) (rate(chat_keypackage_claims_total[5m]))",
          "legendFormat": "claimed: {{served}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(chat_keypackages_published_total[5m]))",
          "legendFormat": "published",
          "refId": "B",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "description": "`last_resort` means somebody's single-use stock has run out: their next joins reuse a KeyPackage, and nothing else says so."
    },
    {
      "type": "timeseries",
      "title": "Attachments",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 16,
        "y": 29,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (direction) (rate(chat_blob_bytes_total[5m]))",
          "legendFormat": "{{direction}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (result) (rate(chat_blob_operations_total{result!=\"ok\"}[5m]))",
          "legendFormat": "{{result}}",
          "refId": "B",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "unit": "Bps"
        },
        "overrides": []
      }
    },
    {
      "type": "row",
      "title": "The machine",
      "gridPos": {
        "x": 0,
        "y": 36,
        "w": 24,
        "h": 1
      },
      "collapsed": false,
      "panels": []
    },
    {
      "type": "timeseries",
      "title": "Database pool",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 0,
        "y": 37,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (instance, state) (chat_db_pool_connections)",
          "legendFormat": "{{instance}} {{state}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      }
    },
    {
      "type": "timeseries",
      "title": "Housekeeping",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 8,
        "y": 37,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (what) (rate(chat_maintenance_swept_total[1h]))",
          "legendFormat": "{{what}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (result) (rate(chat_maintenance_runs_total[6h]))",
          "legendFormat": "runs: {{result}}",
          "refId": "B",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      }
    },
    {
      "type": "timeseries",
      "title": "Scrapes refused",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 16,
        "y": 37,
        "w": 8,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(chat_metrics_scrapes_total{result=\"unauthorized\"}[15m]))",
          "legendFormat": "unauthorized",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "description": "A scraper whose token has drifted, or somebody knocking."
    },
    {
      "type": "row",
      "title": "Bots and the faucet",
      "gridPos": {
        "x": 0,
        "y": 44,
        "w": 24,
        "h": 1
      },
      "collapsed": false,
      "panels": []
    },
    {
      "type": "timeseries",
      "title": "Bot API calls",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 0,
        "y": 45,
        "w": 12,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "topk(10, sum by (method) (rate(chat_bot_api_calls_total[5m])))",
          "legendFormat": "{{method}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(chat_bot_api_calls_total{result=\"refused\"}[5m]))",
          "legendFormat": "refused",
          "refId": "B",
          "editorMode": "code",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(rate(chat_bot_updates_delivered_total[5m]))",
          "legendFormat": "updates delivered",
          "refId": "C",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      }
    },
    {
      "type": "timeseries",
      "title": "Faucet drips",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "x": 12,
        "y": 45,
        "w": 12,
        "h": 7
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (network, result) (rate(chat_faucet_funds_total[30m]))",
          "legendFormat": "{{network}} {{result}}",
          "refId": "A",
          "editorMode": "code",
          "range": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "description": "A faucet that has run dry refuses, and refusing is the only way anybody finds out."
    }
  ]
}