--- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: creationTimestamp: "2026-02-17T12:49:43Z" generation: 1 labels: app.kubernetes.io/component: alert-router app.kubernetes.io/instance: main app.kubernetes.io/managed-by: cluster-monitoring-operator app.kubernetes.io/name: alertmanager app.kubernetes.io/part-of: openshift-monitoring app.kubernetes.io/version: 0.28.1 prometheus: k8s role: alert-rules managedFields: - apiVersion: monitoring.coreos.com/v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app.kubernetes.io/component: {} f:app.kubernetes.io/instance: {} f:app.kubernetes.io/managed-by: {} f:app.kubernetes.io/name: {} f:app.kubernetes.io/part-of: {} f:app.kubernetes.io/version: {} f:prometheus: {} f:role: {} f:spec: .: {} f:groups: .: {} k:{"name":"alertmanager.rules"}: .: {} f:name: {} f:rules: {} manager: operator operation: Update time: "2026-02-17T12:49:43Z" name: alertmanager-main-rules namespace: openshift-monitoring resourceVersion: "9795" uid: 28aee620-d7ed-4c8e-a42a-3b10aee06fc0 spec: groups: - name: alertmanager.rules rules: - alert: AlertmanagerFailedReload annotations: description: Configuration has failed to load for {{ $labels.namespace }}/{{ $labels.pod}}. runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/AlertmanagerFailedReload.md summary: Reloading an Alertmanager configuration has failed. expr: | # Without max_over_time, failed scrapes could create false negatives, see # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. max_over_time(alertmanager_config_last_reload_successful{job=~"alertmanager-main|alertmanager-user-workload"}[5m]) == 0 for: 10m labels: severity: critical - alert: AlertmanagerMembersInconsistent annotations: description: Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} has only found {{ $value }} members of the {{$labels.job}} cluster. summary: A member of an Alertmanager cluster has not found all other cluster members. expr: | # Without max_over_time, failed scrapes could create false negatives, see # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. max_over_time(alertmanager_cluster_members{job=~"alertmanager-main|alertmanager-user-workload"}[5m]) < on (namespace,service) group_left count by (namespace,service) (max_over_time(alertmanager_cluster_members{job=~"alertmanager-main|alertmanager-user-workload"}[5m])) for: 15m labels: severity: warning - alert: AlertmanagerFailedToSendAlerts annotations: description: Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} failed to send {{ $value | humanizePercentage }} of notifications to {{ $labels.integration }}. runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/AlertmanagerFailedToSendAlerts.md summary: An Alertmanager instance failed to send notifications. expr: | ( rate(alertmanager_notifications_failed_total{job=~"alertmanager-main|alertmanager-user-workload"}[15m]) / ignoring (reason) group_left rate(alertmanager_notifications_total{job=~"alertmanager-main|alertmanager-user-workload"}[15m]) ) > 0.01 for: 5m labels: severity: warning - alert: AlertmanagerClusterFailedToSendAlerts annotations: description: The minimum notification failure rate to {{ $labels.integration }} sent from any instance in the {{$labels.job}} cluster is {{ $value | humanizePercentage }}. runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/AlertmanagerClusterFailedToSendAlerts.md summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. expr: | min by (namespace,service, integration) ( rate(alertmanager_notifications_failed_total{job=~"alertmanager-main|alertmanager-user-workload", integration=~`.*`}[15m]) / ignoring (reason) group_left rate(alertmanager_notifications_total{job=~"alertmanager-main|alertmanager-user-workload", integration=~`.*`}[15m]) > 0 ) > 0.01 for: 5m labels: severity: warning - alert: AlertmanagerConfigInconsistent annotations: description: Alertmanager instances within the {{$labels.job}} cluster have different configurations. summary: Alertmanager instances within the same cluster have different configurations. expr: | count by (namespace,service) ( count_values by (namespace,service) ("config_hash", alertmanager_config_hash{job=~"alertmanager-main|alertmanager-user-workload"}) ) != 1 for: 20m labels: severity: warning - alert: AlertmanagerClusterDown annotations: description: '{{ $value | humanizePercentage }} of Alertmanager instances within the {{$labels.job}} cluster have been up for less than half of the last 5m.' summary: Half or more of the Alertmanager instances within the same cluster are down. expr: | ( count by (namespace,service) ( avg_over_time(up{job=~"alertmanager-main|alertmanager-user-workload"}[5m]) < 0.5 ) / count by (namespace,service) ( up{job=~"alertmanager-main|alertmanager-user-workload"} ) ) >= 0.5 for: 5m labels: severity: warning