21 Prometheus alerting rules for Jaeger.Exported via Embedded exporter (v2+), Embedded exporter (legacy, <v2).These rules cover critical and warning conditions — copy and paste the YAML into your Prometheus configuration.
⚠️
Alert thresholds depend on the nature of your applications. Some queries may have arbitrary tolerance thresholds. Building an efficient monitoring platform takes time. 😉
groups:
- name: EmbeddedExporter
rules:
# Fires when all storage operations for a given type are failing and none are succeeding.
# Indicates the storage backend (Cassandra, Elasticsearch, etc.) is likely unreachable or misconfigured.
- alert: JaegerStorageCompletelyUnavailable
expr: sum(rate(jaeger_storage_requests_total{result="err"}[1m])) by (instance, job, namespace, operation) > 0 and sum(rate(jaeger_storage_requests_total{result="ok"}[1m])) by (instance, job, namespace, operation) == 0
for: 2m
labels:
severity: critical
annotations:
summary: Jaeger storage completely unavailable (instance {{ $labels.instance }})
description: "Jaeger on {{ $labels.instance }} has 100% storage errors for {{ $labels.operation }} — storage backend may be down.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
# Fires when an operation (e.g. find_traces, get_services) has received requests but none succeeded.
# May indicate a persistent storage error or a backend that is slow to recover.
- alert: JaegerNoStorageReadsSucceeding
expr: sum(increase(jaeger_storage_requests_total{result="ok"}[15m])) by (instance, job, namespace, operation) == 0 and sum(increase(jaeger_storage_requests_total[15m])) by (instance, job, namespace, operation) > 0
for: 5m
labels:
severity: warning
annotations:
summary: Jaeger no storage reads succeeding (instance {{ $labels.instance }})
description: "Jaeger on {{ $labels.instance }} has no successful storage reads for {{ $labels.operation }} in the past 15 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: JaegerHighStorageErrorRate
expr: 100 * sum(rate(jaeger_storage_requests_total{result="err"}[1m])) by (instance, job, namespace, operation) / sum(rate(jaeger_storage_requests_total[1m])) by (instance, job, namespace, operation) > 1 and sum(rate(jaeger_storage_requests_total[1m])) by (instance, job, namespace, operation) > 0
for: 5m
labels:
severity: warning
annotations:
summary: Jaeger high storage error rate (instance {{ $labels.instance }})
description: "Jaeger on {{ $labels.instance }} is experiencing {{ $value | humanize }}% storage errors on {{ $labels.operation }}.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
# Threshold of 1s is a rough default. Adjust based on your storage backend and data volume.
- alert: JaegerSlowStorageOperations
expr: histogram_quantile(0.99, sum(rate(jaeger_storage_latency_seconds_bucket[5m])) by (le, instance, job, namespace, operation)) > 1
for: 5m
labels:
severity: warning
annotations:
summary: Jaeger slow storage operations (instance {{ $labels.instance }})
description: "Jaeger on {{ $labels.instance }} storage p99 latency for {{ $labels.operation }} is {{ $value | humanizeDuration }}.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
# Filters on http_route="/api/traces" (the trace search endpoint). The http_server_request_duration_seconds
# metric is emitted by the otelhttp middleware used by the Jaeger query service.
- alert: JaegerQueryServiceHighErrorRate
expr: 100 * sum(rate(http_server_request_duration_seconds_count{http_route="/api/traces",http_response_status_code=~"5.."}[1m])) by (instance, job, namespace) / sum(rate(http_server_request_duration_seconds_count{http_route="/api/traces"}[1m])) by (instance, job, namespace) > 1 and sum(rate(http_server_request_duration_seconds_count{http_route="/api/traces"}[1m])) by (instance, job, namespace) > 0
for: 5m
labels:
severity: warning
annotations:
summary: Jaeger query service high error rate (instance {{ $labels.instance }})
description: "Jaeger query service on {{ $labels.instance }} is returning {{ $value | humanize }}% HTTP 5xx errors.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
# Threshold of 2s is a rough default. Adjust based on your storage backend and data volume.
- alert: JaegerQueryServiceSlowResponses
expr: histogram_quantile(0.99, sum(rate(http_server_request_duration_seconds_bucket{http_route="/api/traces"}[5m])) by (le, instance, job, namespace)) > 2
for: 5m
labels:
severity: warning
annotations:
summary: Jaeger query service slow responses (instance {{ $labels.instance }})
description: "Jaeger query service on {{ $labels.instance }} p99 response latency is {{ $value | humanizeDuration }}.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
# Single trace retrieval (/api/traces/{traceID}) can be slower than search, especially for large traces.
# Threshold of 5s is a rough default.
- alert: JaegerSlowSingleTraceRetrieval
expr: histogram_quantile(0.99, sum(rate(http_server_request_duration_seconds_bucket{http_route="/api/traces/{traceID}"}[5m])) by (le, instance, job, namespace)) > 5
for: 5m
labels:
severity: warning
annotations:
summary: Jaeger slow single trace retrieval (instance {{ $labels.instance }})
description: "Jaeger on {{ $labels.instance }} p99 latency for single trace retrieval is {{ $value | humanizeDuration }}.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
# Errors on /api/services indicate the storage backend cannot return the list of instrumented services,
# which breaks the Jaeger UI service selector.
- alert: JaegerServiceDiscoveryErrors
expr: 100 * sum(rate(http_server_request_duration_seconds_count{http_route="/api/services",http_response_status_code=~"5.."}[1m])) by (instance, job, namespace) / sum(rate(http_server_request_duration_seconds_count{http_route="/api/services"}[1m])) by (instance, job, namespace) > 1 and sum(rate(http_server_request_duration_seconds_count{http_route="/api/services"}[1m])) by (instance, job, namespace) > 0
for: 5m
labels:
severity: warning
annotations:
summary: Jaeger service discovery errors (instance {{ $labels.instance }})
description: "Jaeger on {{ $labels.instance }} is returning {{ $value | humanize }}% HTTP 5xx errors on the services endpoint.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
Jaeger v2 is built on OpenTelemetry Collector and exposes metrics on port 8888 (/metrics).
It emits standard otelcol_* pipeline metrics alongside Jaeger-specific storage and query metrics.
For span ingestion pipeline alerts (refused spans, export failures, queue saturation),
use the OpenTelemetry Collector rules instead.
Jaeger on {{ $labels.instance }} has 100% storage errors for {{ $labels.operation }} — storage backend may be down.
# Fires when all storage operations for a given type are failing and none are succeeding.
# Indicates the storage backend (Cassandra, Elasticsearch, etc.) is likely unreachable or misconfigured.
- alert: JaegerStorageCompletelyUnavailable
expr: sum(rate(jaeger_storage_requests_total{result="err"}[1m])) by (instance, job, namespace, operation) > 0 and sum(rate(jaeger_storage_requests_total{result="ok"}[1m])) by (instance, job, namespace, operation) == 0
for: 2m
labels:
severity: critical
annotations:
summary: Jaeger storage completely unavailable (instance {{ $labels.instance }})
description: "Jaeger on {{ $labels.instance }} has 100% storage errors for {{ $labels.operation }} — storage backend may be down.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
Jaeger on {{ $labels.instance }} has no successful storage reads for {{ $labels.operation }} in the past 15 minutes.
# Fires when an operation (e.g. find_traces, get_services) has received requests but none succeeded.
# May indicate a persistent storage error or a backend that is slow to recover.
- alert: JaegerNoStorageReadsSucceeding
expr: sum(increase(jaeger_storage_requests_total{result="ok"}[15m])) by (instance, job, namespace, operation) == 0 and sum(increase(jaeger_storage_requests_total[15m])) by (instance, job, namespace, operation) > 0
for: 5m
labels:
severity: warning
annotations:
summary: Jaeger no storage reads succeeding (instance {{ $labels.instance }})
description: "Jaeger on {{ $labels.instance }} has no successful storage reads for {{ $labels.operation }} in the past 15 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
Jaeger on {{ $labels.instance }} p99 latency for single trace retrieval is {{ $value | humanizeDuration }}.
# Single trace retrieval (/api/traces/{traceID}) can be slower than search, especially for large traces.
# Threshold of 5s is a rough default.
- alert: JaegerSlowSingleTraceRetrieval
expr: histogram_quantile(0.99, sum(rate(http_server_request_duration_seconds_bucket{http_route="/api/traces/{traceID}"}[5m])) by (le, instance, job, namespace)) > 5
for: 5m
labels:
severity: warning
annotations:
summary: Jaeger slow single trace retrieval (instance {{ $labels.instance }})
description: "Jaeger on {{ $labels.instance }} p99 latency for single trace retrieval is {{ $value | humanizeDuration }}.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
These rules target Jaeger v1.x metrics (jaeger_* prefix).
Jaeger v1 reached end-of-life on December 31, 2025.
For Jaeger v2+, use the "Embedded exporter (v2+)" rules instead.
Note: jaeger-agent was deprecated in v1.35 and removed in v2.0.
Jaeger client on {{ $labels.instance }} is experiencing {{ $value | humanize }}% RPC HTTP errors.
# 1% threshold mixes 4xx (client-driven, varies by application) and 5xx (infra-fault, should be ~0%) into one ratio. Adjust based on your 4xx baseline; a pure 5xx spike should still be investigated regardless of this threshold.
- alert: JaegerClientRPCRequestErrors
expr: 100 * sum(rate(jaeger_client_jaeger_rpc_http_requests{status_code=~"4xx|5xx"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_client_jaeger_rpc_http_requests[1m])) by (instance, job, namespace) > 1 and sum(rate(jaeger_client_jaeger_rpc_http_requests[1m])) by (instance, job, namespace) > 0
for: 15m
labels:
severity: warning
annotations:
summary: Jaeger client RPC request errors (instance {{ $labels.instance }})
description: "Jaeger client on {{ $labels.instance }} is experiencing {{ $value | humanize }}% RPC HTTP errors.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
Jaeger agent on {{ $labels.instance }} is dropping UDP packets on its Thrift-compact span ingestion server ({{ $value }} packets/s), meaning spans are lost before they reach the collector.
# 1/s is a rough default; absolute drop count scales with your span ingestion volume — adjust based on your workload.
- alert: JaegerAgentUDPPacketsDropped
expr: rate(jaeger_agent_thrift_udp_server_packets_dropped_total[1m]) > 1
for: 15m
labels:
severity: warning
annotations:
summary: Jaeger agent UDP packets dropped (instance {{ $labels.instance }})
description: "Jaeger agent on {{ $labels.instance }} is dropping UDP packets on its Thrift-compact span ingestion server ({{ $value }} packets/s), meaning spans are lost before they reach the collector.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
Jaeger collector on {{ $labels.instance }} has an internal span queue length of {{ $value }}, a leading indicator that the collector is falling behind on processing before spans start being dropped.
# 1000 is a rough default; queue depth scales with your span ingestion throughput relative to collector processing capacity — adjust based on your workload.
- alert: JaegerCollectorQueueBacklog
expr: jaeger_collector_queue_length > 1000
for: 15m
labels:
severity: warning
annotations:
summary: Jaeger collector queue backlog (instance {{ $labels.instance }})
description: "Jaeger collector on {{ $labels.instance }} has an internal span queue length of {{ $value }}, a leading indicator that the collector is falling behind on processing before spans start being dropped.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
Jaeger collector on {{ $labels.instance }} has a p99 span persistence latency of {{ $value }}s, indicating the storage backend is slow to accept span writes.
# 0.5s threshold is a rough default; write latency depends on your storage backend and span ingestion volume — adjust based on your baseline.
- alert: JaegerCollectorSlowSpanPersistence
expr: histogram_quantile(0.99, sum by (le, instance, job, namespace) (rate(jaeger_collector_save_latency_bucket[5m]))) > 0.5
for: 5m
labels:
severity: warning
annotations:
summary: Jaeger collector slow span persistence (instance {{ $labels.instance }})
description: "Jaeger collector on {{ $labels.instance }} has a p99 span persistence latency of {{ $value }}s, indicating the storage backend is slow to accept span writes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
Jaeger on {{ $labels.instance }} is failing {{ $value | humanize }}% of writes to the Cassandra storage backend, which can lead to permanent span data loss.
- alert: JaegerCassandraWritesFailing
expr: 100 * sum(rate(jaeger_cassandra_errors_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_cassandra_attempts_total[1m])) by (instance, job, namespace) > 1 and sum(rate(jaeger_cassandra_attempts_total[1m])) by (instance, job, namespace) > 0
for: 15m
labels:
severity: warning
annotations:
summary: Jaeger Cassandra writes failing (instance {{ $labels.instance }})
description: "Jaeger on {{ $labels.instance }} is failing {{ $value | humanize }}% of writes to the Cassandra storage backend, which can lead to permanent span data loss.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
Jaeger on {{ $labels.instance }} is failing {{ $value | humanize }}% of reads from the Cassandra storage backend, which impacts trace query availability.
- alert: JaegerCassandraReadsFailing
expr: 100 * sum(rate(jaeger_cassandra_read_errors_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_cassandra_read_attempts_total[1m])) by (instance, job, namespace) > 1 and sum(rate(jaeger_cassandra_read_attempts_total[1m])) by (instance, job, namespace) > 0
for: 15m
labels:
severity: warning
annotations:
summary: Jaeger Cassandra reads failing (instance {{ $labels.instance }})
description: "Jaeger on {{ $labels.instance }} is failing {{ $value | humanize }}% of reads from the Cassandra storage backend, which impacts trace query availability.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
Jaeger Prometheus Alert Rules