OpenTelemetry Collector Prometheus Alert Rules

groups:
- name: EmbeddedExporter
  rules:
      # Adjust the job label regex to match the actual job name in your Prometheus scrape config.
    - alert: OpenTelemetryCollectorDown
      expr: up{job=~".*otel.*collector.*"} == 0
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: OpenTelemetry Collector down (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector instance has disappeared or is not being scraped\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 0.05/s avoids firing on transient single-event spikes.
    - alert: OpenTelemetryCollectorReceiverRefusedSpans
      expr: rate(otelcol_receiver_refused_spans[5m]) > 0.05
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: OpenTelemetry Collector receiver refused spans (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector is refusing {{ $value | humanize }}/s spans on {{ $labels.receiver }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 0.05/s avoids firing on transient single-event spikes.
    - alert: OpenTelemetryCollectorReceiverRefusedMetricPoints
      expr: rate(otelcol_receiver_refused_metric_points[5m]) > 0.05
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: OpenTelemetry Collector receiver refused metric points (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector is refusing {{ $value | humanize }}/s metric points on {{ $labels.receiver }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 0.05/s avoids firing on transient single-event spikes.
    - alert: OpenTelemetryCollectorReceiverRefusedLogRecords
      expr: rate(otelcol_receiver_refused_log_records[5m]) > 0.05
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: OpenTelemetry Collector receiver refused log records (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector is refusing {{ $value | humanize }}/s log records on {{ $labels.receiver }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 0.05/s avoids firing on transient single-event spikes.
    - alert: OpenTelemetryCollectorExporterFailedSpans
      expr: rate(otelcol_exporter_send_failed_spans[5m]) > 0.05
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: OpenTelemetry Collector exporter failed spans (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector failing to send {{ $value | humanize }}/s spans via {{ $labels.exporter }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 0.05/s avoids firing on transient single-event spikes.
    - alert: OpenTelemetryCollectorExporterFailedMetricPoints
      expr: rate(otelcol_exporter_send_failed_metric_points[5m]) > 0.05
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: OpenTelemetry Collector exporter failed metric points (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector failing to send {{ $value | humanize }}/s metric points via {{ $labels.exporter }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 0.05/s avoids firing on transient single-event spikes.
    - alert: OpenTelemetryCollectorExporterFailedLogRecords
      expr: rate(otelcol_exporter_send_failed_log_records[5m]) > 0.05
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: OpenTelemetry Collector exporter failed log records (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector failing to send {{ $value | humanize }}/s log records via {{ $labels.exporter }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: OpenTelemetryCollectorExporterQueueNearlyFull
      expr: (otelcol_exporter_queue_size / on(instance, job, exporter) otelcol_exporter_queue_capacity) > 0.8 and otelcol_exporter_queue_capacity > 0
      for: 0m
      labels:
        severity: warning
      annotations:
        summary: OpenTelemetry Collector exporter queue nearly full (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector exporter {{ $labels.exporter }} queue is over 80% full\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 0.05/s avoids firing on transient single-event spikes.
      # These processor metrics are deprecated since collector v0.110.0.
    - alert: OpenTelemetryCollectorProcessorRefusedSpans
      expr: rate(otelcol_processor_refused_spans[5m]) > 0.05
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: OpenTelemetry Collector processor refused spans (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector processor {{ $labels.processor }} is refusing spans ({{ $value | humanize }}/s), likely due to backpressure.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 0.05/s avoids firing on transient single-event spikes.
      # These processor metrics are deprecated since collector v0.110.0.
    - alert: OpenTelemetryCollectorProcessorRefusedMetricPoints
      expr: rate(otelcol_processor_refused_metric_points[5m]) > 0.05
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: OpenTelemetry Collector processor refused metric points (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector processor {{ $labels.processor }} is refusing metric points ({{ $value | humanize }}/s), likely due to backpressure.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: OpenTelemetryCollectorHighMemoryUsage
      expr: (otelcol_process_runtime_heap_alloc_bytes / on(instance, job) otelcol_process_runtime_total_sys_memory_bytes) > 0.9
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: OpenTelemetry Collector high memory usage (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector memory usage is above 90%\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: OpenTelemetryCollectorOTLPReceiverErrors
      expr: rate(otelcol_receiver_accepted_spans{receiver=~"otlp"}[5m]) == 0 and rate(otelcol_receiver_refused_spans{receiver=~"otlp"}[5m]) > 0
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: OpenTelemetry Collector OTLP receiver errors (instance {{ $labels.instance }})
        description: "OpenTelemetry Collector OTLP receiver is completely failing - all spans are being refused\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

OpenTelemetry Collector self-monitoring metrics are exposed on port 8888 by default at the /metrics endpoint.
These alerts monitor the collector's health when metrics are ingested via the Prometheus OTLP endpoint or scraped directly.
All collector internal metrics are prefixed with 'otelcol_'.

wget https://raw.githubusercontent.com/samber/awesome-prometheus-alerts/refs/heads/master/dist/rules/opentelemetry-collector/embedded-exporter.yml

  # Adjust the job label regex to match the actual job name in your Prometheus scrape config.
- alert: OpenTelemetryCollectorDown
  expr: up{job=~".*otel.*collector.*"} == 0
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: OpenTelemetry Collector down (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector instance has disappeared or is not being scraped\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 0.05/s avoids firing on transient single-event spikes.
- alert: OpenTelemetryCollectorReceiverRefusedSpans
  expr: rate(otelcol_receiver_refused_spans[5m]) > 0.05
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: OpenTelemetry Collector receiver refused spans (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector is refusing {{ $value | humanize }}/s spans on {{ $labels.receiver }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 0.05/s avoids firing on transient single-event spikes.
- alert: OpenTelemetryCollectorReceiverRefusedMetricPoints
  expr: rate(otelcol_receiver_refused_metric_points[5m]) > 0.05
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: OpenTelemetry Collector receiver refused metric points (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector is refusing {{ $value | humanize }}/s metric points on {{ $labels.receiver }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 0.05/s avoids firing on transient single-event spikes.
- alert: OpenTelemetryCollectorReceiverRefusedLogRecords
  expr: rate(otelcol_receiver_refused_log_records[5m]) > 0.05
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: OpenTelemetry Collector receiver refused log records (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector is refusing {{ $value | humanize }}/s log records on {{ $labels.receiver }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 0.05/s avoids firing on transient single-event spikes.
- alert: OpenTelemetryCollectorExporterFailedSpans
  expr: rate(otelcol_exporter_send_failed_spans[5m]) > 0.05
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: OpenTelemetry Collector exporter failed spans (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector failing to send {{ $value | humanize }}/s spans via {{ $labels.exporter }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 0.05/s avoids firing on transient single-event spikes.
- alert: OpenTelemetryCollectorExporterFailedMetricPoints
  expr: rate(otelcol_exporter_send_failed_metric_points[5m]) > 0.05
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: OpenTelemetry Collector exporter failed metric points (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector failing to send {{ $value | humanize }}/s metric points via {{ $labels.exporter }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 0.05/s avoids firing on transient single-event spikes.
- alert: OpenTelemetryCollectorExporterFailedLogRecords
  expr: rate(otelcol_exporter_send_failed_log_records[5m]) > 0.05
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: OpenTelemetry Collector exporter failed log records (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector failing to send {{ $value | humanize }}/s log records via {{ $labels.exporter }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: OpenTelemetryCollectorExporterQueueNearlyFull
  expr: (otelcol_exporter_queue_size / on(instance, job, exporter) otelcol_exporter_queue_capacity) > 0.8 and otelcol_exporter_queue_capacity > 0
  for: 0m
  labels:
    severity: warning
  annotations:
    summary: OpenTelemetry Collector exporter queue nearly full (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector exporter {{ $labels.exporter }} queue is over 80% full\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 0.05/s avoids firing on transient single-event spikes.
  # These processor metrics are deprecated since collector v0.110.0.
- alert: OpenTelemetryCollectorProcessorRefusedSpans
  expr: rate(otelcol_processor_refused_spans[5m]) > 0.05
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: OpenTelemetry Collector processor refused spans (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector processor {{ $labels.processor }} is refusing spans ({{ $value | humanize }}/s), likely due to backpressure.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 0.05/s avoids firing on transient single-event spikes.
  # These processor metrics are deprecated since collector v0.110.0.
- alert: OpenTelemetryCollectorProcessorRefusedMetricPoints
  expr: rate(otelcol_processor_refused_metric_points[5m]) > 0.05
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: OpenTelemetry Collector processor refused metric points (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector processor {{ $labels.processor }} is refusing metric points ({{ $value | humanize }}/s), likely due to backpressure.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: OpenTelemetryCollectorHighMemoryUsage
  expr: (otelcol_process_runtime_heap_alloc_bytes / on(instance, job) otelcol_process_runtime_total_sys_memory_bytes) > 0.9
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: OpenTelemetry Collector high memory usage (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector memory usage is above 90%\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: OpenTelemetryCollectorOTLPReceiverErrors
  expr: rate(otelcol_receiver_accepted_spans{receiver=~"otlp"}[5m]) == 0 and rate(otelcol_receiver_refused_spans{receiver=~"otlp"}[5m]) > 0
  for: 2m
  labels:
    severity: critical
  annotations:
    summary: OpenTelemetry Collector OTLP receiver errors (instance {{ $labels.instance }})
    description: "OpenTelemetry Collector OTLP receiver is completely failing - all spans are being refused\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"