apiVersion: monitoring.coreos.com/v1
kind: Probe
metadata:
  name: <namespace>-<env>-<endpoint>-probe
  namespace: monitoring
  labels:
    release: kube-prometheus-stack
spec:
  jobName: <namespace>-<env>-<endpoint>-probe
  interval: 30s
  scrapeTimeout: 15s
  module: http_2xx
  prober:
    url: blackbox-exporter.monitoring.svc.cluster.local:9115
    scheme: http
    path: /probe
  targets:
    staticConfig:
      static:
        - https://<endpoint>
---              
apiVersion: v1
kind: ConfigMap
metadata:
  name: <namespace>-<env>-<endpoint>-dashboard
  namespace: monitoring
  labels:
    grafana_dashboard: "1"
data:
  <namespace>-<env>-<endpoint>-dashboard.json: |
    {
      "annotations": {
        "list": []
      },
      "editable": true,
      "fiscalYearStartMonth": 0,
      "graphTooltip": 1,
      "links": [],
      "panels": [
        {
          "type": "stat",
          "title": "Services UP",
          "id": 1,
          "gridPos": {
            "h": 5,
            "w": 6,
            "x": 0,
            "y": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "targets": [
            {
              "refId": "A",
              "expr": "sum(probe_success{job=\"<namespace>-<env>-<endpoint>-probe\",instance=~\"$instance\"})"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "short",
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "red",
                    "value": null
                  },
                  {
                    "color": "green",
                    "value": 1
                  }
                ]
              }
            }
          },
          "options": {
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "orientation": "auto",
            "textMode": "auto",
            "colorMode": "value",
            "graphMode": "none"
          }
        },
        {
          "type": "stat",
          "title": "Services DOWN",
          "id": 2,
          "gridPos": {
            "h": 5,
            "w": 6,
            "x": 6,
            "y": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "targets": [
            {
              "refId": "A",
              "expr": "count(probe_success{job=\"<namespace>-<env>-<endpoint>-probe\",instance=~\"$instance\"}) - sum(probe_success{job=\"<namespace>-<env>-<endpoint>-probe\",instance=~\"$instance\"})"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "short",
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 1
                  }
                ]
              }
            }
          },
          "options": {
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "orientation": "auto",
            "textMode": "auto",
            "colorMode": "value",
            "graphMode": "none"
          }
        },
        {
          "type": "stat",
          "title": "Availability 24h",
          "id": 3,
          "gridPos": {
            "h": 5,
            "w": 6,
            "x": 12,
            "y": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "targets": [
            {
              "refId": "A",
              "expr": "avg(avg_over_time(probe_success{job=\"<namespace>-<env>-<endpoint>-probe\",instance=~\"$instance\"}[24h])) * 100"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "percent",
              "min": 0,
              "max": 100,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "red",
                    "value": null
                  },
                  {
                    "color": "orange",
                    "value": 99
                  },
                  {
                    "color": "green",
                    "value": 99.9
                  }
                ]
              }
            }
          },
          "options": {
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "orientation": "auto",
            "textMode": "auto",
            "colorMode": "value",
            "graphMode": "area"
          }
        },
        {
          "type": "stat",
          "title": "Avg Latency",
          "id": 4,
          "gridPos": {
            "h": 5,
            "w": 6,
            "x": 18,
            "y": 0
          },
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "targets": [
            {
              "refId": "A",
              "expr": "avg(probe_duration_seconds{job=\"<namespace>-<env>-<endpoint>-probe\",instance=~\"$instance\"}) * 1000"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "ms",
              "decimals": 0
            }
          },
          "options": {
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            },
            "orientation": "auto",
            "textMode": "auto",
            "colorMode": "value",
            "graphMode": "none"
          }
        },
        {
          "type": "table",
          "title": "HTTP Services Status",
          "id": 5,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 5
          },
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "targets": [
            {
              "refId": "A",
              "expr": "probe_success{job=\"<namespace>-<env>-<endpoint>-probe\",instance=~\"$instance\"}",
              "instant": true,
              "format": "table"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "custom": {
                "align": "auto",
                "cellOptions": {
                  "type": "color-text"
                }
              }
            }
          },
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "renameByName": {
                  "Value": "Status",
                  "instance": "Service"
                }
              }
            }
          ],
          "options": {
            "showHeader": true,
            "cellHeight": "sm"
          }
        },
        {
          "type": "table",
          "title": "HTTP Status Code",
          "id": 6,
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 5
          },
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "targets": [
            {
              "refId": "A",
              "expr": "probe_http_status_code{job=\"<namespace>-<env>-<endpoint>-probe\",instance=~\"$instance\"}",
              "instant": true,
              "format": "table"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "short",
              "custom": {
                "align": "auto",
                "cellOptions": {
                  "type": "color-text"
                }
              }
            }
          },
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "renameByName": {
                  "Value": "HTTP Status",
                  "instance": "Service"
                }
              }
            }
          ],
          "options": {
            "showHeader": true,
            "cellHeight": "sm"
          }
        },
        {
          "type": "timeseries",
          "title": "HTTP Response Time",
          "id": 7,
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 13
          },
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "targets": [
            {
              "refId": "A",
              "expr": "probe_duration_seconds{job=\"<namespace>-<env>-<endpoint>-probe\",instance=~\"$instance\"} * 1000",
              "legendFormat": "{{instance}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "ms",
              "decimals": 0
            }
          },
          "options": {
            "legend": {
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "desc"
            }
          }
        },
        {
          "type": "table",
          "title": "Currently DOWN",
          "id": 8,
          "gridPos": {
            "h": 7,
            "w": 12,
            "x": 0,
            "y": 22
          },
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "targets": [
            {
              "refId": "A",
              "expr": "probe_success{job=\"<namespace>-<env>-<endpoint>-probe\",instance=~\"$instance\"} == 0",
              "instant": true,
              "format": "table"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "custom": {
                "cellOptions": {
                  "type": "color-text"
                }
              }
            }
          },
          "transformations": [
            {
              "id": "organize",
              "options": {
                "excludeByName": {
                  "Time": true
                },
                "renameByName": {
                  "Value": "Status",
                  "instance": "Service"
                }
              }
            }
          ],
          "options": {
            "showHeader": true,
            "cellHeight": "sm"
          }
        },
        {
          "type": "timeseries",
          "title": "Availability 24h",
          "id": 9,
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 12,
            "y": 22
          },
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "targets": [
            {
              "refId": "A",
              "expr": "avg_over_time(probe_success{job=\"<namespace>-<env>-<endpoint>-probe\",instance=~\"$instance\"}[24h]) * 100",
              "legendFormat": "{{instance}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "percent",
              "min": 0,
              "max": 100,
              "decimals": 2
            }
          },
          "options": {
            "legend": {
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "desc"
            }
          }
        },
        {
          "type": "timeseries",
          "title": "P95 Latency - 1h",
          "id": 10,
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 31
          },
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "targets": [
            {
              "refId": "A",
              "expr": "quantile_over_time(0.95, probe_duration_seconds{job=\"<namespace>-<env>-<endpoint>-probe\",instance=~\"$instance\"}[1h]) * 1000",
              "legendFormat": "{{instance}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "ms",
              "decimals": 0
            }
          },
          "options": {
            "legend": {
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "desc"
            }
          }
        }
      ],
      "refresh": "30s",
      "schemaVersion": 41,
      "tags": [
        "blackbox",
        "http",
        "monitoring"
      ],

      "templating": {
        "list": [
          {
            "name": "instance",
            "label": "Service",
            "type": "query",
            "datasource": {
              "type": "prometheus",
              "uid": "prometheus"
            },
            "definition": "label_values(probe_success{job=\"<namespace>-<env>-<endpoint>-probe\"}, instance)",
            "query": {
              "query": "label_values(probe_success{job=\"<namespace>-<env>-<endpoint>-probe\"}, instance)",
              "refId": "StandardVariableQuery"
            },
            "includeAll": true,
            "multi": true,
            "allValue": ".*",
            "refresh": 1,
            "sort": 1
          }
        ]
      },
      "time": {
        "from": "now-24h",
        "to": "now"
      },
      "timepicker": {},
      "timezone": "browser",
      "title": "<namespace>-<env>-<endpoint>-dashboard",
      "uid": "<namespace>-<env>-<endpoint>-dashboard",
      "version": 1,
      "weekStart": ""
    }