Grafana Tempo Prometheus Alert Rules

groups:
- name: EmbeddedExporter
  rules:
    - alert: TempoLiveStoreUnhealthy
      expr: max by (job) (tempo_ring_members{state="Unhealthy", name="live-store"}) > 0
      for: 15m
      labels:
        severity: critical
      annotations:
        summary: Tempo live store unhealthy (instance {{ $labels.instance }})
        description: "Tempo has {{ $value }} unhealthy live store(s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoMetricsGeneratorUnhealthy
      expr: max by (job) (tempo_ring_members{state="Unhealthy", name="metrics-generator"}) > 0
      for: 15m
      labels:
        severity: critical
      annotations:
        summary: Tempo metrics generator unhealthy (instance {{ $labels.instance }})
        description: "Tempo has {{ $value }} unhealthy metrics generator(s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Uses a two-window approach: 1h for historical count and 5m to confirm the issue is ongoing.
    - alert: TempoCompactionsFailing
      expr: sum by (job) (increase(tempodb_compaction_errors_total[1h])) > 2 and sum by (job) (increase(tempodb_compaction_errors_total[5m])) > 0
      for: 1h
      labels:
        severity: critical
      annotations:
        summary: Tempo compactions failing (instance {{ $labels.instance }})
        description: "{{ $value }} compactions have failed in the past hour.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoPollsFailing
      expr: sum by (job) (increase(tempodb_blocklist_poll_errors_total[1h])) > 2 and sum by (job) (increase(tempodb_blocklist_poll_errors_total[5m])) > 0
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: Tempo polls failing (instance {{ $labels.instance }})
        description: "{{ $value }} blocklist polls have failed in the past hour.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoTenantIndexFailures
      expr: sum by (job) (increase(tempodb_blocklist_tenant_index_errors_total[1h])) > 2 and sum by (job) (increase(tempodb_blocklist_tenant_index_errors_total[5m])) > 0
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: Tempo tenant index failures (instance {{ $labels.instance }})
        description: "{{ $value }} tenant index failures in the past hour.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoNoTenantIndexBuilders
      expr: sum by (tenant) (tempodb_blocklist_tenant_index_builder) == 0 and on() max(tempodb_blocklist_length) > 0
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: Tempo no tenant index builders (instance {{ $labels.instance }})
        description: "No tenant index builders for tenant {{ $labels.tenant }}. Tenant index will quickly become stale.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 600s (10 minutes). Adjust based on your tenant index build interval.
    - alert: TempoTenantIndexTooOld
      expr: max by (tenant) (tempodb_blocklist_tenant_index_age_seconds) > 600
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: Tempo tenant index too old (instance {{ $labels.instance }})
        description: "Tenant index for {{ $labels.tenant }} is {{ $value }}s old.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Fires when the blocklist grows more than 40% over 7 days.
    - alert: TempoBlockListRisingQuickly
      expr: (avg(tempodb_blocklist_length) / avg(tempodb_blocklist_length offset 7d) - 1) * 100 > 40 and avg(tempodb_blocklist_length offset 7d) > 0
      for: 15m
      labels:
        severity: critical
      annotations:
        summary: Tempo block list rising quickly (instance {{ $labels.instance }})
        description: "Tempo blocklist length is up {{ printf \"%.0f\" $value }}% over the last 7 days. Consider scaling compactors.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 250 blocks per compactor instance. Adjust based on your environment. If
      # you run a split backend-scheduler/backend-worker architecture, consider normalizing
      # this count by the number of backend-worker instances before thresholding, since
      # sensitivity would otherwise scale inversely with fleet size.
    - alert: TempoCompactionTooManyOutstandingBlocksCritical
      expr: sum by (instance) (tempodb_compaction_outstanding_blocks) > 250
      for: 24h
      labels:
        severity: critical
      annotations:
        summary: Tempo compaction too many outstanding blocks critical (instance {{ $labels.instance }})
        description: "There are too many outstanding compaction blocks for {{ $labels.instance }}. Increase compactor resources immediately.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 0.05/s avoids firing on transient single-event spikes.
    - alert: TempoDistributorUsageTrackerErrors
      expr: sum by (job, reason) (rate(tempo_distributor_usage_tracker_errors_total[5m])) > 0.05
      for: 30m
      labels:
        severity: critical
      annotations:
        summary: Tempo distributor usage tracker errors (instance {{ $labels.instance }})
        description: "Tempo distributor usage tracker errors for {{ $labels.job }} at {{ $value | humanize }}/s (reason {{ $labels.reason }}).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoMetricsGeneratorProcessorUpdatesFailing
      expr: sum by (job) (increase(tempo_metrics_generator_active_processors_update_failed_total[5m])) > 2
      for: 15m
      labels:
        severity: critical
      annotations:
        summary: Tempo metrics generator processor updates failing (instance {{ $labels.instance }})
        description: "Tempo metrics generator processor updates are failing for {{ $labels.job }} ({{ $value }} failures in 5m).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoMetricsGeneratorCollectionsFailing
      expr: sum by (job) (increase(tempo_metrics_generator_registry_collections_failed_total[5m])) > 2
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: Tempo metrics generator collections failing (instance {{ $labels.instance }})
        description: "Tempo metrics generator collections are failing for {{ $labels.job }} ({{ $value }} failures in 5m).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoDistributorKafkaProduceFailures
      expr: 100 * sum by (job) (rate(tempo_distributor_produce_failures_total{reason!="cancelled-before-producing"}[5m])) / sum by (job) (rate(tempo_distributor_produce_records_total[5m])) > 0.1 and sum by (job) (rate(tempo_distributor_produce_records_total[5m])) > 0
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: Tempo distributor Kafka produce failures (instance {{ $labels.instance }})
        description: "Tempo distributor {{ $labels.job }} is failing to produce {{ printf \"%.2f\" $value }}% of trace records to Kafka, an early indicator of write-path trace loss.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # 300s threshold is a rough default; Kafka lag scales with your ingestion throughput and consumer processing speed — adjust based on your workload.
    - alert: TempoLiveStoreAllOwnersLagging
      expr: min by (job, partition) (tempo_ingest_group_partition_lag_seconds{container="live-store"}) > 300
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: Tempo live store all owners lagging (instance {{ $labels.instance }})
        description: "All live-store owners of Tempo partition {{ $labels.partition }} are lagging by {{ $value }}s behind their Kafka ingest partition, causing a partial read outage for recent traces.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # 1200s threshold is a rough default; Kafka lag scales with your ingestion throughput and consumer processing speed — adjust based on your workload.
    - alert: TempoLiveStorePartitionLagCritical
      expr: tempo_ingest_group_partition_lag_seconds{container="live-store"} > 1200
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: Tempo live store partition lag critical (instance {{ $labels.instance }})
        description: "Tempo live-store {{ $labels.instance }} (group {{ $labels.group }}) is lagging by {{ $value }}s behind Kafka ingest partition {{ $labels.partition }}; recent traces on this partition may be significantly delayed.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoLiveStorePartitionsUnowned
      expr: max by (job) (tempo_partition_ring_partitions{name="livestore-partitions", state=~"Active|Inactive"}) > count(count by (partition, job) (tempo_live_store_partition_owned)) by (job)
      for: 10m
      labels:
        severity: critical
      annotations:
        summary: Tempo live store partitions unowned (instance {{ $labels.instance }})
        description: "{{ $labels.job }} has live-store partitions with no owner in the partition ring; spans for those partitions are fully unqueryable.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # 300s threshold is a rough default; Kafka lag scales with your ingestion throughput and consumer processing speed — adjust based on your workload.
    - alert: TempoBlockBuilderPartitionLagCritical
      expr: tempo_ingest_group_partition_lag_seconds{container="block-builder"} > 300
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: Tempo block builder partition lag critical (instance {{ $labels.instance }})
        description: "Tempo block-builder {{ $labels.instance }} (group {{ $labels.group }}) is lagging by {{ $value }}s behind Kafka ingest partition {{ $labels.partition }}; traces are significantly delayed in being flushed into durable blocks.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoBackendSchedulerJobsFailing
      expr: 100 * sum by (job) (increase(tempo_backend_scheduler_jobs_failed_total[5m])) / sum by (job) (increase(tempo_backend_scheduler_jobs_created_total[5m])) > 5 and sum by (job) (increase(tempo_backend_scheduler_jobs_created_total[5m])) > 0
      for: 10m
      labels:
        severity: critical
      annotations:
        summary: Tempo backend scheduler jobs failing (instance {{ $labels.instance }})
        description: "Tempo backend-scheduler job failure rate is {{ printf \"%.2f\" $value }}% for {{ $labels.job }}, indicating compaction jobs are failing in the backend-scheduler/backend-worker pipeline.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # 900s threshold is a rough default; Kafka lag scales with your ingestion throughput and consumer processing speed — adjust based on your workload.
    - alert: TempoMetricsGeneratorPartitionLagCritical
      expr: tempo_ingest_group_partition_lag_seconds{container="metrics-generator"} > 900
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: Tempo metrics generator partition lag critical (instance {{ $labels.instance }})
        description: "Tempo metrics-generator {{ $labels.instance }} (group {{ $labels.group }}) is lagging by {{ $value }}s behind Kafka ingest partition {{ $labels.partition }}; generated service-graph and span metrics reflect stale data.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoBlockBuildersPartitionsMismatch
      expr: max by (job) (tempo_partition_ring_partitions{name="livestore-partitions", state=~"Active|Inactive"}) > sum by (job) (tempo_block_builder_owned_partitions)
      for: 10m
      labels:
        severity: critical
      annotations:
        summary: Tempo block builders partitions mismatch (instance {{ $labels.instance }})
        description: "{{ $labels.job }} has more live-store partitions in the partition ring than block-builders currently own, indicating stuck or misconfigured partition ownership.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoDistributorUnhealthy
      expr: max by (job) (tempo_ring_members{state="Unhealthy", name="distributor"}) > 0
      for: 15m
      labels:
        severity: warning
      annotations:
        summary: Tempo distributor unhealthy (instance {{ $labels.instance }})
        description: "Tempo has {{ $value }} unhealthy distributor(s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 100 blocks per compactor instance. Adjust based on your environment.
    - alert: TempoCompactionTooManyOutstandingBlocksWarning
      expr: sum by (instance) (tempodb_compaction_outstanding_blocks) > 100
      for: 6h
      labels:
        severity: warning
      annotations:
        summary: Tempo compaction too many outstanding blocks warning (instance {{ $labels.instance }})
        description: "There are too many outstanding compaction blocks for {{ $labels.instance }}. Consider increasing compactor resources.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoMetricsGeneratorServiceGraphsDroppingSpans
      expr: 100 * sum by (job) (rate(tempo_metrics_generator_processor_service_graphs_dropped_spans_total[5m])) / sum by (job) (rate(tempo_metrics_generator_spans_received_total[5m])) > 0.5 and sum by (job) (rate(tempo_metrics_generator_spans_received_total[5m])) > 0
      for: 15m
      labels:
        severity: warning
      annotations:
        summary: Tempo metrics generator service graphs dropping spans (instance {{ $labels.instance }})
        description: "Tempo metrics generator is dropping {{ printf \"%.2f\" $value }}% of spans in service graphs for {{ $labels.job }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Fires when the memcached error rate exceeds 20%. Only relevant if Tempo is configured with memcached caching.
    - alert: TempoMemcachedErrorsElevated
      expr: 100 * sum by (name, job) (rate(tempo_memcache_request_duration_seconds_count{status_code="500"}[5m])) / sum by (name, job) (rate(tempo_memcache_request_duration_seconds_count[5m])) > 20 and sum by (name, job) (rate(tempo_memcache_request_duration_seconds_count[5m])) > 0
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: Tempo memcached errors elevated (instance {{ $labels.instance }})
        description: "Tempo memcached error rate is {{ printf \"%.2f\" $value }}% for {{ $labels.name }} in {{ $labels.job }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # 200s threshold is a rough default; Kafka lag scales with your ingestion throughput and consumer processing speed — adjust based on your workload.
    - alert: TempoLiveStorePartitionLagWarning
      expr: tempo_ingest_group_partition_lag_seconds{container="live-store"} > 200
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: Tempo live store partition lag warning (instance {{ $labels.instance }})
        description: "Tempo live-store {{ $labels.instance }} (group {{ $labels.group }}) is lagging by {{ $value }}s behind Kafka ingest partition {{ $labels.partition }}, degrading read completeness for recent traces.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # The block-builder consumes in batches, so its raw lag sawtooths up to consume_cycle_duration
      # (5m by default) on every healthy cycle. The 6m average is what makes 200s meaningful: it
      # means the block-builder is burning roughly 45% of its cycle budget. Raise the window
      # alongside consume_cycle_duration if you tune it.
    - alert: TempoBlockBuilderPartitionLagWarning
      expr: avg_over_time(tempo_ingest_group_partition_lag_seconds{container="block-builder"}[6m]) > 200
      for: 0m
      labels:
        severity: warning
      annotations:
        summary: Tempo block builder partition lag warning (instance {{ $labels.instance }})
        description: "Tempo block-builder {{ $labels.instance }} (group {{ $labels.group }}) is lagging by {{ $value }}s behind Kafka ingest partition {{ $labels.partition }}; traces are delayed in being flushed into durable blocks.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoBadOverrides
      expr: sum by (job) (tempo_runtime_config_last_reload_successful == 0) > 0
      for: 15m
      labels:
        severity: critical
      annotations:
        summary: Tempo bad overrides (instance {{ $labels.instance }})
        description: "{{ $labels.job }} failed to reload runtime overrides.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: TempoUserConfigurableOverridesReloadFailing
      expr: sum by (job) (increase(tempo_overrides_user_configurable_overrides_reload_failed_total[1h])) > 5 and sum by (job) (increase(tempo_overrides_user_configurable_overrides_reload_failed_total[5m])) > 0
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: Tempo user configurable overrides reload failing (instance {{ $labels.instance }})
        description: "{{ $value }} user-configurable overrides reloads have failed in the past hour.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

Most rules below apply to any Tempo deployment. A subset (live-store health, and any
rule mentioning distributor Kafka-produce, block-builder, or backend-scheduler
lag/ownership) instead requires Tempo v3.0+'s Kafka-based ingest architecture
(distributor -> Kafka -> block-builder/live-store). On a classic/monolithic Tempo
deployment those specific metrics won't exist and their rules will simply stay inactive.

wget https://raw.githubusercontent.com/samber/awesome-prometheus-alerts/refs/heads/master/dist/rules/grafana-tempo/embedded-exporter.yml

- alert: TempoLiveStoreUnhealthy
  expr: max by (job) (tempo_ring_members{state="Unhealthy", name="live-store"}) > 0
  for: 15m
  labels:
    severity: critical
  annotations:
    summary: Tempo live store unhealthy (instance {{ $labels.instance }})
    description: "Tempo has {{ $value }} unhealthy live store(s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoMetricsGeneratorUnhealthy
  expr: max by (job) (tempo_ring_members{state="Unhealthy", name="metrics-generator"}) > 0
  for: 15m
  labels:
    severity: critical
  annotations:
    summary: Tempo metrics generator unhealthy (instance {{ $labels.instance }})
    description: "Tempo has {{ $value }} unhealthy metrics generator(s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Uses a two-window approach: 1h for historical count and 5m to confirm the issue is ongoing.
- alert: TempoCompactionsFailing
  expr: sum by (job) (increase(tempodb_compaction_errors_total[1h])) > 2 and sum by (job) (increase(tempodb_compaction_errors_total[5m])) > 0
  for: 1h
  labels:
    severity: critical
  annotations:
    summary: Tempo compactions failing (instance {{ $labels.instance }})
    description: "{{ $value }} compactions have failed in the past hour.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoPollsFailing
  expr: sum by (job) (increase(tempodb_blocklist_poll_errors_total[1h])) > 2 and sum by (job) (increase(tempodb_blocklist_poll_errors_total[5m])) > 0
  for: 0m
  labels:
    severity: critical
  annotations:
    summary: Tempo polls failing (instance {{ $labels.instance }})
    description: "{{ $value }} blocklist polls have failed in the past hour.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoTenantIndexFailures
  expr: sum by (job) (increase(tempodb_blocklist_tenant_index_errors_total[1h])) > 2 and sum by (job) (increase(tempodb_blocklist_tenant_index_errors_total[5m])) > 0
  for: 0m
  labels:
    severity: critical
  annotations:
    summary: Tempo tenant index failures (instance {{ $labels.instance }})
    description: "{{ $value }} tenant index failures in the past hour.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoNoTenantIndexBuilders
  expr: sum by (tenant) (tempodb_blocklist_tenant_index_builder) == 0 and on() max(tempodb_blocklist_length) > 0
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: Tempo no tenant index builders (instance {{ $labels.instance }})
    description: "No tenant index builders for tenant {{ $labels.tenant }}. Tenant index will quickly become stale.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 600s (10 minutes). Adjust based on your tenant index build interval.
- alert: TempoTenantIndexTooOld
  expr: max by (tenant) (tempodb_blocklist_tenant_index_age_seconds) > 600
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: Tempo tenant index too old (instance {{ $labels.instance }})
    description: "Tenant index for {{ $labels.tenant }} is {{ $value }}s old.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Fires when the blocklist grows more than 40% over 7 days.
- alert: TempoBlockListRisingQuickly
  expr: (avg(tempodb_blocklist_length) / avg(tempodb_blocklist_length offset 7d) - 1) * 100 > 40 and avg(tempodb_blocklist_length offset 7d) > 0
  for: 15m
  labels:
    severity: critical
  annotations:
    summary: Tempo block list rising quickly (instance {{ $labels.instance }})
    description: "Tempo blocklist length is up {{ printf \"%.0f\" $value }}% over the last 7 days. Consider scaling compactors.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 250 blocks per compactor instance. Adjust based on your environment. If
  # you run a split backend-scheduler/backend-worker architecture, consider normalizing
  # this count by the number of backend-worker instances before thresholding, since
  # sensitivity would otherwise scale inversely with fleet size.
- alert: TempoCompactionTooManyOutstandingBlocksCritical
  expr: sum by (instance) (tempodb_compaction_outstanding_blocks) > 250
  for: 24h
  labels:
    severity: critical
  annotations:
    summary: Tempo compaction too many outstanding blocks critical (instance {{ $labels.instance }})
    description: "There are too many outstanding compaction blocks for {{ $labels.instance }}. Increase compactor resources immediately.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 0.05/s avoids firing on transient single-event spikes.
- alert: TempoDistributorUsageTrackerErrors
  expr: sum by (job, reason) (rate(tempo_distributor_usage_tracker_errors_total[5m])) > 0.05
  for: 30m
  labels:
    severity: critical
  annotations:
    summary: Tempo distributor usage tracker errors (instance {{ $labels.instance }})
    description: "Tempo distributor usage tracker errors for {{ $labels.job }} at {{ $value | humanize }}/s (reason {{ $labels.reason }}).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoMetricsGeneratorProcessorUpdatesFailing
  expr: sum by (job) (increase(tempo_metrics_generator_active_processors_update_failed_total[5m])) > 2
  for: 15m
  labels:
    severity: critical
  annotations:
    summary: Tempo metrics generator processor updates failing (instance {{ $labels.instance }})
    description: "Tempo metrics generator processor updates are failing for {{ $labels.job }} ({{ $value }} failures in 5m).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoMetricsGeneratorCollectionsFailing
  expr: sum by (job) (increase(tempo_metrics_generator_registry_collections_failed_total[5m])) > 2
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: Tempo metrics generator collections failing (instance {{ $labels.instance }})
    description: "Tempo metrics generator collections are failing for {{ $labels.job }} ({{ $value }} failures in 5m).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoDistributorKafkaProduceFailures
  expr: 100 * sum by (job) (rate(tempo_distributor_produce_failures_total{reason!="cancelled-before-producing"}[5m])) / sum by (job) (rate(tempo_distributor_produce_records_total[5m])) > 0.1 and sum by (job) (rate(tempo_distributor_produce_records_total[5m])) > 0
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: Tempo distributor Kafka produce failures (instance {{ $labels.instance }})
    description: "Tempo distributor {{ $labels.job }} is failing to produce {{ printf \"%.2f\" $value }}% of trace records to Kafka, an early indicator of write-path trace loss.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # 300s threshold is a rough default; Kafka lag scales with your ingestion throughput and consumer processing speed — adjust based on your workload.
- alert: TempoLiveStoreAllOwnersLagging
  expr: min by (job, partition) (tempo_ingest_group_partition_lag_seconds{container="live-store"}) > 300
  for: 2m
  labels:
    severity: critical
  annotations:
    summary: Tempo live store all owners lagging (instance {{ $labels.instance }})
    description: "All live-store owners of Tempo partition {{ $labels.partition }} are lagging by {{ $value }}s behind their Kafka ingest partition, causing a partial read outage for recent traces.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # 1200s threshold is a rough default; Kafka lag scales with your ingestion throughput and consumer processing speed — adjust based on your workload.
- alert: TempoLiveStorePartitionLagCritical
  expr: tempo_ingest_group_partition_lag_seconds{container="live-store"} > 1200
  for: 2m
  labels:
    severity: critical
  annotations:
    summary: Tempo live store partition lag critical (instance {{ $labels.instance }})
    description: "Tempo live-store {{ $labels.instance }} (group {{ $labels.group }}) is lagging by {{ $value }}s behind Kafka ingest partition {{ $labels.partition }}; recent traces on this partition may be significantly delayed.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoLiveStorePartitionsUnowned
  expr: max by (job) (tempo_partition_ring_partitions{name="livestore-partitions", state=~"Active|Inactive"}) > count(count by (partition, job) (tempo_live_store_partition_owned)) by (job)
  for: 10m
  labels:
    severity: critical
  annotations:
    summary: Tempo live store partitions unowned (instance {{ $labels.instance }})
    description: "{{ $labels.job }} has live-store partitions with no owner in the partition ring; spans for those partitions are fully unqueryable.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # 300s threshold is a rough default; Kafka lag scales with your ingestion throughput and consumer processing speed — adjust based on your workload.
- alert: TempoBlockBuilderPartitionLagCritical
  expr: tempo_ingest_group_partition_lag_seconds{container="block-builder"} > 300
  for: 2m
  labels:
    severity: critical
  annotations:
    summary: Tempo block builder partition lag critical (instance {{ $labels.instance }})
    description: "Tempo block-builder {{ $labels.instance }} (group {{ $labels.group }}) is lagging by {{ $value }}s behind Kafka ingest partition {{ $labels.partition }}; traces are significantly delayed in being flushed into durable blocks.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoBackendSchedulerJobsFailing
  expr: 100 * sum by (job) (increase(tempo_backend_scheduler_jobs_failed_total[5m])) / sum by (job) (increase(tempo_backend_scheduler_jobs_created_total[5m])) > 5 and sum by (job) (increase(tempo_backend_scheduler_jobs_created_total[5m])) > 0
  for: 10m
  labels:
    severity: critical
  annotations:
    summary: Tempo backend scheduler jobs failing (instance {{ $labels.instance }})
    description: "Tempo backend-scheduler job failure rate is {{ printf \"%.2f\" $value }}% for {{ $labels.job }}, indicating compaction jobs are failing in the backend-scheduler/backend-worker pipeline.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # 900s threshold is a rough default; Kafka lag scales with your ingestion throughput and consumer processing speed — adjust based on your workload.
- alert: TempoMetricsGeneratorPartitionLagCritical
  expr: tempo_ingest_group_partition_lag_seconds{container="metrics-generator"} > 900
  for: 2m
  labels:
    severity: critical
  annotations:
    summary: Tempo metrics generator partition lag critical (instance {{ $labels.instance }})
    description: "Tempo metrics-generator {{ $labels.instance }} (group {{ $labels.group }}) is lagging by {{ $value }}s behind Kafka ingest partition {{ $labels.partition }}; generated service-graph and span metrics reflect stale data.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoBlockBuildersPartitionsMismatch
  expr: max by (job) (tempo_partition_ring_partitions{name="livestore-partitions", state=~"Active|Inactive"}) > sum by (job) (tempo_block_builder_owned_partitions)
  for: 10m
  labels:
    severity: critical
  annotations:
    summary: Tempo block builders partitions mismatch (instance {{ $labels.instance }})
    description: "{{ $labels.job }} has more live-store partitions in the partition ring than block-builders currently own, indicating stuck or misconfigured partition ownership.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoDistributorUnhealthy
  expr: max by (job) (tempo_ring_members{state="Unhealthy", name="distributor"}) > 0
  for: 15m
  labels:
    severity: warning
  annotations:
    summary: Tempo distributor unhealthy (instance {{ $labels.instance }})
    description: "Tempo has {{ $value }} unhealthy distributor(s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 100 blocks per compactor instance. Adjust based on your environment.
- alert: TempoCompactionTooManyOutstandingBlocksWarning
  expr: sum by (instance) (tempodb_compaction_outstanding_blocks) > 100
  for: 6h
  labels:
    severity: warning
  annotations:
    summary: Tempo compaction too many outstanding blocks warning (instance {{ $labels.instance }})
    description: "There are too many outstanding compaction blocks for {{ $labels.instance }}. Consider increasing compactor resources.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoMetricsGeneratorServiceGraphsDroppingSpans
  expr: 100 * sum by (job) (rate(tempo_metrics_generator_processor_service_graphs_dropped_spans_total[5m])) / sum by (job) (rate(tempo_metrics_generator_spans_received_total[5m])) > 0.5 and sum by (job) (rate(tempo_metrics_generator_spans_received_total[5m])) > 0
  for: 15m
  labels:
    severity: warning
  annotations:
    summary: Tempo metrics generator service graphs dropping spans (instance {{ $labels.instance }})
    description: "Tempo metrics generator is dropping {{ printf \"%.2f\" $value }}% of spans in service graphs for {{ $labels.job }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Fires when the memcached error rate exceeds 20%. Only relevant if Tempo is configured with memcached caching.
- alert: TempoMemcachedErrorsElevated
  expr: 100 * sum by (name, job) (rate(tempo_memcache_request_duration_seconds_count{status_code="500"}[5m])) / sum by (name, job) (rate(tempo_memcache_request_duration_seconds_count[5m])) > 20 and sum by (name, job) (rate(tempo_memcache_request_duration_seconds_count[5m])) > 0
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: Tempo memcached errors elevated (instance {{ $labels.instance }})
    description: "Tempo memcached error rate is {{ printf \"%.2f\" $value }}% for {{ $labels.name }} in {{ $labels.job }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # 200s threshold is a rough default; Kafka lag scales with your ingestion throughput and consumer processing speed — adjust based on your workload.
- alert: TempoLiveStorePartitionLagWarning
  expr: tempo_ingest_group_partition_lag_seconds{container="live-store"} > 200
  for: 2m
  labels:
    severity: warning
  annotations:
    summary: Tempo live store partition lag warning (instance {{ $labels.instance }})
    description: "Tempo live-store {{ $labels.instance }} (group {{ $labels.group }}) is lagging by {{ $value }}s behind Kafka ingest partition {{ $labels.partition }}, degrading read completeness for recent traces.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # The block-builder consumes in batches, so its raw lag sawtooths up to consume_cycle_duration
  # (5m by default) on every healthy cycle. The 6m average is what makes 200s meaningful: it
  # means the block-builder is burning roughly 45% of its cycle budget. Raise the window
  # alongside consume_cycle_duration if you tune it.
- alert: TempoBlockBuilderPartitionLagWarning
  expr: avg_over_time(tempo_ingest_group_partition_lag_seconds{container="block-builder"}[6m]) > 200
  for: 0m
  labels:
    severity: warning
  annotations:
    summary: Tempo block builder partition lag warning (instance {{ $labels.instance }})
    description: "Tempo block-builder {{ $labels.instance }} (group {{ $labels.group }}) is lagging by {{ $value }}s behind Kafka ingest partition {{ $labels.partition }}; traces are delayed in being flushed into durable blocks.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoBadOverrides
  expr: sum by (job) (tempo_runtime_config_last_reload_successful == 0) > 0
  for: 15m
  labels:
    severity: critical
  annotations:
    summary: Tempo bad overrides (instance {{ $labels.instance }})
    description: "{{ $labels.job }} failed to reload runtime overrides.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: TempoUserConfigurableOverridesReloadFailing
  expr: sum by (job) (increase(tempo_overrides_user_configurable_overrides_reload_failed_total[1h])) > 5 and sum by (job) (increase(tempo_overrides_user_configurable_overrides_reload_failed_total[5m])) > 0
  for: 0m
  labels:
    severity: critical
  annotations:
    summary: Tempo user configurable overrides reload failing (instance {{ $labels.instance }})
    description: "{{ $value }} user-configurable overrides reloads have failed in the past hour.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"