Ceph Prometheus Alert Rules

groups:
- name: EmbeddedExporter
  rules:
    - alert: CephOSDDown(>=10%)
      expr: count(ceph_osd_up == 0) / count(ceph_osd_up) * 100 >= 10
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: Ceph OSD down (>= 10%) (instance {{ $labels.instance }})
        description: "{{ $value | humanize }}% of OSDs are down on the cluster, exceeding the safe threshold and putting data availability at risk.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephPGDown
      expr: ceph_pg_down > 0
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: Ceph PG down (instance {{ $labels.instance }})
        description: "Some Ceph placement groups are down. Please ensure that all the data are available.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephMonitorQuorumAtRisk
      expr: (ceph_health_detail{name="MON_DOWN"} == 1) * on() group_right() (count(ceph_mon_quorum_status == 1) == bool (floor(count(ceph_mon_metadata) / 2) + 1)) == 1
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: Ceph monitor quorum at risk (instance {{ $labels.instance }})
        description: "Ceph monitor quorum is at the minimum required to stay operational; losing one more monitor will make the cluster inoperable for all clients.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephOSDDown
      expr: ceph_osd_up == 0
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: Ceph OSD Down (instance {{ $labels.instance }})
        description: "Ceph Object Storage Daemon Down\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephMonitorDown
      expr: count(ceph_mon_quorum_status == 0) <= (count(ceph_mon_metadata) - floor(count(ceph_mon_metadata) / 2 + 1))
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: Ceph monitor down (instance {{ $labels.instance }})
        description: "One or more Ceph monitors are down while quorum is still intact; losing an additional monitor would make the cluster inoperable.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephMonitorLowSpace
      expr: ceph_health_detail{name="MON_DISK_LOW"} == 1
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: Ceph monitor low space (instance {{ $labels.instance }})
        description: "Ceph monitor storage is low.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Ceph internally triggers OSD_NEARFULL based on the nearfull_ratio (default 85%).
      # ceph_health_detail exposes named health checks as individual time series.
    - alert: CephOSDNearFull
      expr: ceph_health_detail{name="OSD_NEARFULL"} == 1
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: Ceph OSD near full (instance {{ $labels.instance }})
        description: "A Ceph OSD is dangerously full. Please add more disks.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephOSDReweighted
      expr: ceph_osd_weight < 1
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: Ceph OSD reweighted (instance {{ $labels.instance }})
        description: "Ceph Object Storage Daemon takes too much time to resize.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephPGBackfillFull
      expr: ceph_pg_backfill_toofull > 0
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: Ceph PG backfill full (instance {{ $labels.instance }})
        description: "Some Ceph placement groups are located on full Object Storage Daemon on cluster. Those PGs can be unavailable shortly. Please check OSDs, change weight or reconfigure CRUSH rules.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephMonitorClockSkew
      expr: ceph_health_detail{name="MON_CLOCK_SKEW"} == 1
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: Ceph monitor clock skew (instance {{ $labels.instance }})
        description: "Ceph monitor clock skew detected. Please check ntp and hardware clock settings\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephHealthError
      expr: ceph_health_status == 2
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: Ceph health error (instance {{ $labels.instance }})
        description: "Ceph cluster is in HEALTH_ERR state\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephPGIncomplete
      expr: ceph_pg_incomplete > 0
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: Ceph PG incomplete (instance {{ $labels.instance }})
        description: "Some Ceph placement groups are incomplete. Please ensure that all the data are available.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephPGUnavailable
      expr: ceph_pg_total - ceph_pg_active > 0
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: Ceph PG unavailable (instance {{ $labels.instance }})
        description: "Some Ceph placement groups are unavailable.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephOSDFull
      expr: ceph_health_detail{name="OSD_FULL"} == 1
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: Ceph OSD full (instance {{ $labels.instance }})
        description: "One or more Ceph OSDs have reached the full ratio; writes to the affected pools are blocked until space is freed or OSDs are added.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephDaemonCrashDetected
      expr: ceph_health_detail{name="RECENT_CRASH"} == 1
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: Ceph daemon crash detected (instance {{ $labels.instance }})
        description: "One or more Ceph daemons (OSD, mon, mgr, mds...) have crashed recently and the crash has not been acknowledged. Investigate and archive with 'ceph crash archive <id>'.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephMDSUnhealthy
      expr: ceph_health_detail{name=~"MDS_DAMAGE|MDS_ALL_DOWN|FS_DEGRADED|FS_WITH_FAILED_MDS|MDS_HEALTH_READ_ONLY"} == 1
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: Ceph MDS unhealthy (instance {{ $labels.instance }})
        description: "The CephFS filesystem is damaged, fully offline, degraded, has a failed MDS rank with no standby, or has switched to read-only mode. Data may be partially or fully inaccessible; check 'ceph fs status' for details.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephHealthWarning
      expr: ceph_health_status == 1
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: Ceph health warning (instance {{ $labels.instance }})
        description: "Ceph cluster is in HEALTH_WARN state\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 5000ms (5 seconds). Adjust based on your expected OSD performance.
    - alert: CephHighOSDLatency
      expr: ceph_osd_apply_latency_ms > 5000
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: Ceph high OSD latency (instance {{ $labels.instance }})
        description: "Ceph Object Storage Daemon latency is high. Please check if it doesn't stuck in weird state.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephPGInconsistent
      expr: ceph_pg_inconsistent > 0
      for: 0m
      labels:
        severity: warning
      annotations:
        summary: Ceph PG inconsistent (instance {{ $labels.instance }})
        description: "Some Ceph placement groups are inconsistent. Data is available but inconsistent across nodes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephPGActivationLong
      expr: ceph_pg_activating > 0
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: Ceph PG activation long (instance {{ $labels.instance }})
        description: "Some Ceph placement groups are too long to activate.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephPoolNearFull
      expr: ceph_health_detail{name="POOL_NEAR_FULL"} == 1
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: Ceph pool near full (instance {{ $labels.instance }})
        description: "A Ceph pool has exceeded its near-full threshold. Writes may still succeed, but the pool risks going read-only if more capacity isn't made available soon.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephPoolBackfillFull
      expr: ceph_health_detail{name="POOL_BACKFILLFULL"} == 1
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: Ceph pool backfill full (instance {{ $labels.instance }})
        description: "A Ceph pool is approaching its near-full threshold, which will prevent recovery/backfill operations from completing until more capacity is added.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephPGNotScrubbed
      expr: ceph_health_detail{name="PG_NOT_SCRUBBED"} == 1
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: Ceph PG not scrubbed (instance {{ $labels.instance }})
        description: "One or more placement groups have missed their scrub interval, which checks metadata integrity and consistency across replicas. Run 'ceph pg scrub <pgid>' to trigger manually.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephPGNotDeepScrubbed
      expr: ceph_health_detail{name="PG_NOT_DEEP_SCRUBBED"} == 1
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: Ceph PG not deep scrubbed (instance {{ $labels.instance }})
        description: "One or more placement groups have missed their deep scrub interval, which compares data replicas to protect against silent bit-rot.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephSlowOps
      expr: ceph_healthcheck_slow_ops > 0
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: Ceph slow ops (instance {{ $labels.instance }})
        description: "{{ $value }} OSD/mon requests are taking longer than the configured complaint time to complete, an early symptom of client-facing latency and cluster degradation.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: CephPoolFull
      expr: ceph_health_detail{name="POOL_FULL"} == 1
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: Ceph pool full (instance {{ $labels.instance }})
        description: "A Ceph pool has reached its quota, or the OSDs backing it have reached the full threshold; writes to the pool are blocked until quota is increased or capacity is added.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

wget https://raw.githubusercontent.com/samber/awesome-prometheus-alerts/refs/heads/master/dist/rules/ceph/embedded-exporter.yml

- alert: CephOSDDown(>=10%)
  expr: count(ceph_osd_up == 0) / count(ceph_osd_up) * 100 >= 10
  for: 0m
  labels:
    severity: critical
  annotations:
    summary: Ceph OSD down (>= 10%) (instance {{ $labels.instance }})
    description: "{{ $value | humanize }}% of OSDs are down on the cluster, exceeding the safe threshold and putting data availability at risk.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephPGDown
  expr: ceph_pg_down > 0
  for: 0m
  labels:
    severity: critical
  annotations:
    summary: Ceph PG down (instance {{ $labels.instance }})
    description: "Some Ceph placement groups are down. Please ensure that all the data are available.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephMonitorQuorumAtRisk
  expr: (ceph_health_detail{name="MON_DOWN"} == 1) * on() group_right() (count(ceph_mon_quorum_status == 1) == bool (floor(count(ceph_mon_metadata) / 2) + 1)) == 1
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: Ceph monitor quorum at risk (instance {{ $labels.instance }})
    description: "Ceph monitor quorum is at the minimum required to stay operational; losing one more monitor will make the cluster inoperable for all clients.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephOSDDown
  expr: ceph_osd_up == 0
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: Ceph OSD Down (instance {{ $labels.instance }})
    description: "Ceph Object Storage Daemon Down\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephMonitorDown
  expr: count(ceph_mon_quorum_status == 0) <= (count(ceph_mon_metadata) - floor(count(ceph_mon_metadata) / 2 + 1))
  for: 2m
  labels:
    severity: warning
  annotations:
    summary: Ceph monitor down (instance {{ $labels.instance }})
    description: "One or more Ceph monitors are down while quorum is still intact; losing an additional monitor would make the cluster inoperable.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephMonitorLowSpace
  expr: ceph_health_detail{name="MON_DISK_LOW"} == 1
  for: 2m
  labels:
    severity: warning
  annotations:
    summary: Ceph monitor low space (instance {{ $labels.instance }})
    description: "Ceph monitor storage is low.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Ceph internally triggers OSD_NEARFULL based on the nearfull_ratio (default 85%).
  # ceph_health_detail exposes named health checks as individual time series.
- alert: CephOSDNearFull
  expr: ceph_health_detail{name="OSD_NEARFULL"} == 1
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: Ceph OSD near full (instance {{ $labels.instance }})
    description: "A Ceph OSD is dangerously full. Please add more disks.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephOSDReweighted
  expr: ceph_osd_weight < 1
  for: 2m
  labels:
    severity: warning
  annotations:
    summary: Ceph OSD reweighted (instance {{ $labels.instance }})
    description: "Ceph Object Storage Daemon takes too much time to resize.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephPGBackfillFull
  expr: ceph_pg_backfill_toofull > 0
  for: 2m
  labels:
    severity: warning
  annotations:
    summary: Ceph PG backfill full (instance {{ $labels.instance }})
    description: "Some Ceph placement groups are located on full Object Storage Daemon on cluster. Those PGs can be unavailable shortly. Please check OSDs, change weight or reconfigure CRUSH rules.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephMonitorClockSkew
  expr: ceph_health_detail{name="MON_CLOCK_SKEW"} == 1
  for: 2m
  labels:
    severity: warning
  annotations:
    summary: Ceph monitor clock skew (instance {{ $labels.instance }})
    description: "Ceph monitor clock skew detected. Please check ntp and hardware clock settings\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephHealthError
  expr: ceph_health_status == 2
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: Ceph health error (instance {{ $labels.instance }})
    description: "Ceph cluster is in HEALTH_ERR state\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephPGIncomplete
  expr: ceph_pg_incomplete > 0
  for: 0m
  labels:
    severity: critical
  annotations:
    summary: Ceph PG incomplete (instance {{ $labels.instance }})
    description: "Some Ceph placement groups are incomplete. Please ensure that all the data are available.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephPGUnavailable
  expr: ceph_pg_total - ceph_pg_active > 0
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: Ceph PG unavailable (instance {{ $labels.instance }})
    description: "Some Ceph placement groups are unavailable.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephOSDFull
  expr: ceph_health_detail{name="OSD_FULL"} == 1
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: Ceph OSD full (instance {{ $labels.instance }})
    description: "One or more Ceph OSDs have reached the full ratio; writes to the affected pools are blocked until space is freed or OSDs are added.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephDaemonCrashDetected
  expr: ceph_health_detail{name="RECENT_CRASH"} == 1
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: Ceph daemon crash detected (instance {{ $labels.instance }})
    description: "One or more Ceph daemons (OSD, mon, mgr, mds...) have crashed recently and the crash has not been acknowledged. Investigate and archive with 'ceph crash archive <id>'.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephMDSUnhealthy
  expr: ceph_health_detail{name=~"MDS_DAMAGE|MDS_ALL_DOWN|FS_DEGRADED|FS_WITH_FAILED_MDS|MDS_HEALTH_READ_ONLY"} == 1
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: Ceph MDS unhealthy (instance {{ $labels.instance }})
    description: "The CephFS filesystem is damaged, fully offline, degraded, has a failed MDS rank with no standby, or has switched to read-only mode. Data may be partially or fully inaccessible; check 'ceph fs status' for details.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephHealthWarning
  expr: ceph_health_status == 1
  for: 1m
  labels:
    severity: warning
  annotations:
    summary: Ceph health warning (instance {{ $labels.instance }})
    description: "Ceph cluster is in HEALTH_WARN state\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 5000ms (5 seconds). Adjust based on your expected OSD performance.
- alert: CephHighOSDLatency
  expr: ceph_osd_apply_latency_ms > 5000
  for: 1m
  labels:
    severity: warning
  annotations:
    summary: Ceph high OSD latency (instance {{ $labels.instance }})
    description: "Ceph Object Storage Daemon latency is high. Please check if it doesn't stuck in weird state.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephPGInconsistent
  expr: ceph_pg_inconsistent > 0
  for: 0m
  labels:
    severity: warning
  annotations:
    summary: Ceph PG inconsistent (instance {{ $labels.instance }})
    description: "Some Ceph placement groups are inconsistent. Data is available but inconsistent across nodes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephPGActivationLong
  expr: ceph_pg_activating > 0
  for: 2m
  labels:
    severity: warning
  annotations:
    summary: Ceph PG activation long (instance {{ $labels.instance }})
    description: "Some Ceph placement groups are too long to activate.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephPoolNearFull
  expr: ceph_health_detail{name="POOL_NEAR_FULL"} == 1
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: Ceph pool near full (instance {{ $labels.instance }})
    description: "A Ceph pool has exceeded its near-full threshold. Writes may still succeed, but the pool risks going read-only if more capacity isn't made available soon.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephPoolBackfillFull
  expr: ceph_health_detail{name="POOL_BACKFILLFULL"} == 1
  for: 2m
  labels:
    severity: warning
  annotations:
    summary: Ceph pool backfill full (instance {{ $labels.instance }})
    description: "A Ceph pool is approaching its near-full threshold, which will prevent recovery/backfill operations from completing until more capacity is added.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephPGNotScrubbed
  expr: ceph_health_detail{name="PG_NOT_SCRUBBED"} == 1
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: Ceph PG not scrubbed (instance {{ $labels.instance }})
    description: "One or more placement groups have missed their scrub interval, which checks metadata integrity and consistency across replicas. Run 'ceph pg scrub <pgid>' to trigger manually.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephPGNotDeepScrubbed
  expr: ceph_health_detail{name="PG_NOT_DEEP_SCRUBBED"} == 1
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: Ceph PG not deep scrubbed (instance {{ $labels.instance }})
    description: "One or more placement groups have missed their deep scrub interval, which compares data replicas to protect against silent bit-rot.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephSlowOps
  expr: ceph_healthcheck_slow_ops > 0
  for: 1m
  labels:
    severity: warning
  annotations:
    summary: Ceph slow ops (instance {{ $labels.instance }})
    description: "{{ $value }} OSD/mon requests are taking longer than the configured complaint time to complete, an early symptom of client-facing latency and cluster degradation.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: CephPoolFull
  expr: ceph_health_detail{name="POOL_FULL"} == 1
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: Ceph pool full (instance {{ $labels.instance }})
    description: "A Ceph pool has reached its quota, or the OSDs backing it have reached the full threshold; writes to the pool are blocked until quota is increased or capacity is added.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"