GitLab CI Prometheus Alert Rules

groups:
- name: GitlabBuiltInExporter
  rules:
    - alert: GitlabPumaWorkersNotRunning
      expr: 'puma_running_workers < puma_workers'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab Puma workers not running (instance {{ $labels.instance }})
        description: "GitLab Puma on {{ $labels.instance }} has {{ $value }} running workers out of expected total.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 2GB may need adjustment based on your instance size.
      # High memory usage can lead to OOM kills and service disruptions.
    - alert: GitlabHighMemoryUsage
      expr: 'process_resident_memory_bytes{job=~".*gitlab.*"} > 2e+9'
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: GitLab high memory usage (instance {{ $labels.instance }})
        description: "GitLab process on {{ $labels.instance }} is using {{ $value | humanize1024 }}B of RSS memory.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Heap fragmentation above 50% means a significant amount of memory is wasted.
      # A Puma worker restart may help reclaim memory.
    - alert: GitlabRubyHeapFragmentation
      expr: 'ruby_gc_stat_ext_heap_fragmentation{job=~".*gitlab.*"} > 0.5'
      for: 15m
      labels:
        severity: warning
      annotations:
        summary: GitLab Ruby heap fragmentation (instance {{ $labels.instance }})
        description: "GitLab Ruby heap fragmentation on {{ $labels.instance }} is {{ $value }}. High fragmentation wastes memory.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Queued connections indicate Puma workers are saturated.
      # Consider increasing puma['worker_processes'] or puma['max_threads'] in gitlab.rb.
    - alert: GitlabPumaHighQueuedConnections
      expr: 'puma_queued_connections > 5'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab Puma high queued connections (instance {{ $labels.instance }})
        description: "GitLab Puma has {{ $value }} queued connections on {{ $labels.instance }}. Requests are waiting for an available worker thread.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # When the pool is near saturation, requests may block waiting for a connection.
      # Increase db_pool_size in gitlab.rb or investigate slow queries.
    - alert: GitlabDatabaseConnectionPoolSaturation
      expr: 'gitlab_database_connection_pool_busy / gitlab_database_connection_pool_size * 100 > 90 and gitlab_database_connection_pool_size > 0'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab database connection pool saturation (instance {{ $labels.instance }})
        description: "GitLab database connection pool on {{ $labels.instance }} ({{ $labels.class }}) is {{ $value }}% busy.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: GitlabDatabaseConnectionPoolDeadConnections
      expr: 'gitlab_database_connection_pool_dead > 0'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab database connection pool dead connections (instance {{ $labels.instance }})
        description: "GitLab database connection pool on {{ $labels.instance }} ({{ $labels.class }}) has {{ $value }} dead connections.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: GitlabDatabaseConnectionPoolWaiting
      expr: 'gitlab_database_connection_pool_waiting > 0'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab database connection pool waiting (instance {{ $labels.instance }})
        description: "GitLab on {{ $labels.instance }} has {{ $value }} threads waiting for a database connection.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: GitlabHighFileDescriptorUsage
      expr: 'process_open_fds{job=~".*gitlab.*"} / process_max_fds * 100 > 80 and process_max_fds > 0'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab high file descriptor usage (instance {{ $labels.instance }})
        description: "GitLab on {{ $labels.instance }} is using {{ $value }}% of available file descriptors.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: GitlabPumaHighThreadPoolUtilization
      expr: 'puma_active_connections / puma_max_threads * 100 > 90 and puma_max_threads > 0'
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: GitLab Puma high thread pool utilization (instance {{ $labels.instance }})
        description: "GitLab Puma thread pool on {{ $labels.instance }} is {{ $value }}% utilized, indicating sustained saturation before it becomes fully blocked.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: GitlabPumaNoAvailablePoolCapacity
      expr: 'puma_pool_capacity == 0'
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: GitLab Puma no available pool capacity (instance {{ $labels.instance }})
        description: "GitLab Puma pool capacity on {{ $labels.instance }} has been at 0 for 5 minutes. All threads are busy.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold is 5% of all requests returning server errors.
      # Check GitLab logs at /var/log/gitlab/ for root cause.
    - alert: GitlabHighHttpErrorRate
      expr: 'sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) * 100 > 5 and sum(rate(http_requests_total[5m])) > 0'
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: GitLab high HTTP error rate (instance {{ $labels.instance }})
        description: "GitLab is returning more than 5% HTTP 5xx errors on {{ $labels.instance }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 10s may need adjustment based on your instance size and workload.
    - alert: GitlabHighHttpRequestLatency
      expr: 'histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le)) > 10'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab high HTTP request latency (instance {{ $labels.instance }})
        description: "GitLab p95 HTTP request latency on {{ $labels.instance }} is above 10 seconds.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # This metric requires the emit_sidekiq_histogram_metrics feature flag to be enabled.
      # A sustained failure rate indicates background processing issues.
    - alert: GitlabSidekiqJobsFailing
      expr: 'rate(sidekiq_jobs_failed_total[5m]) > 0.1'
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: GitLab Sidekiq jobs failing (instance {{ $labels.instance }})
        description: "GitLab Sidekiq jobs are failing at a rate of {{ $value }} per second on {{ $labels.instance }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # When running jobs approach the concurrency limit, new jobs will queue up.
      # Consider scaling Sidekiq workers or increasing concurrency.
    - alert: GitlabSidekiqQueueTooLarge
      expr: 'sum(sidekiq_running_jobs) >= sum(sidekiq_concurrency) * 0.9'
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: GitLab Sidekiq queue too large (instance {{ $labels.instance }})
        description: "GitLab Sidekiq has {{ $value }} running jobs, approaching concurrency limit on {{ $labels.instance }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # This metric requires the emit_sidekiq_histogram_metrics feature flag to be enabled.
      # GitLab's largest finite completion bucket is 300 seconds, so p95 cannot exceed it.
      # This rule measures the percentage of completed jobs above that boundary, not an interpolated p95.
    - alert: GitlabSidekiqHighJobCompletionTime
      expr: '100 * (sum by (worker) (rate(sidekiq_jobs_completion_seconds_bucket{le="+Inf"}[5m])) - sum by (worker) (rate(sidekiq_jobs_completion_seconds_bucket{le="300"}[5m]))) / sum by (worker) (rate(sidekiq_jobs_completion_seconds_bucket{le="+Inf"}[5m])) > 5'
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: GitLab Sidekiq high job completion time (instance {{ $labels.instance }})
        description: "More than 5% of observed GitLab Sidekiq jobs take longer than 300 seconds ({{ $value }}%).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # This metric requires the emit_sidekiq_histogram_metrics feature flag to be enabled.
      # High queue latency means jobs are stuck waiting. Check Sidekiq concurrency and queue sizes.
      # GitLab's largest finite queue-duration bucket is 60 seconds, so p95 cannot exceed it.
      # This rule measures the percentage of observed waits above that boundary, not an interpolated p95.
    - alert: GitlabSidekiqHighQueueLatency
      expr: '100 * (sum(rate(sidekiq_jobs_queue_duration_seconds_bucket{le="+Inf"}[5m])) - sum(rate(sidekiq_jobs_queue_duration_seconds_bucket{le="60"}[5m]))) / sum(rate(sidekiq_jobs_queue_duration_seconds_bucket{le="+Inf"}[5m])) > 5'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab Sidekiq high queue latency (instance {{ $labels.instance }})
        description: "More than 5% of observed GitLab Sidekiq queue waits take longer than 60 seconds ({{ $value }}%).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # 30s threshold is a rough default; depends on your pipeline complexity and CI instance load — adjust based on your baseline.
    - alert: GitlabCiPipelineCreationSlow
      expr: 'histogram_quantile(0.95, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[5m])) by (le)) > 30'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab CI pipeline creation slow (instance {{ $labels.instance }})
        description: "GitLab CI pipeline creation p95 latency on {{ $labels.instance }} is above 30 seconds.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # This metric may not exist in all GitLab versions. Verify against your GitLab installation.
    - alert: GitlabCiPipelineFailuresIncreasing
      expr: 'deriv(gitlab_ci_pipeline_failure_reasons[5m]) > 0.05'
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: GitLab CI pipeline failures increasing (instance {{ $labels.instance }})
        description: "GitLab CI pipeline failures are increasing on {{ $labels.instance }} ({{ $value }}/s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: GitlabRackUncaughtErrors
      expr: 'rate(rack_uncaught_errors_total[5m]) > 0.05'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab rack uncaught errors (instance {{ $labels.instance }})
        description: "GitLab is experiencing uncaught errors in the Rack layer on {{ $labels.instance }} ({{ $value }}/s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # This may happen during a rolling deployment. If it persists, investigate incomplete upgrades.
    - alert: GitlabVersionMismatch
      expr: 'count(count by (version) (gitlab_build_info)) > 1'
      for: 0m
      labels:
        severity: warning
      annotations:
        summary: GitLab version mismatch (instance {{ $labels.instance }})
        description: "Multiple GitLab versions are running across the fleet.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: GitlabRubyThreadsSaturated
      expr: 'sum by (instance) (gitlab_ruby_threads_running_threads) > on(instance) gitlab_ruby_threads_max_expected_threads * 1.5'
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: GitLab Ruby threads saturated (instance {{ $labels.instance }})
        description: "GitLab running threads on {{ $labels.instance }} have exceeded the expected maximum ({{ $value }}).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: GitlabSidekiqQueueNotDraining
      expr: 'sum by (name) (sidekiq_queue_size) > 0'
      for: 30m
      labels:
        severity: warning
      annotations:
        summary: GitLab Sidekiq queue not draining (instance {{ $labels.instance }})
        description: "GitLab Sidekiq queue {{ $labels.name }} has had a persistent backlog of {{ $value }} jobs for 30 minutes, suggesting a stuck or crashed worker class rather than transient load.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Frequent runner auth failures may indicate expired tokens or misconfigured runners.
    - alert: GitlabCiRunnerAuthenticationFailures
      expr: 'increase(gitlab_ci_runner_authentication_failure_total[5m]) > 5'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab CI runner authentication failures (instance {{ $labels.instance }})
        description: "GitLab CI runners are experiencing authentication failures on {{ $labels.instance }} ({{ $value }} failures).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

wget https://raw.githubusercontent.com/samber/awesome-prometheus-alerts/refs/heads/master/dist/rules/gitlab-ci/gitlab-built-in-exporter.yml

- alert: GitlabPumaWorkersNotRunning
  expr: 'puma_running_workers < puma_workers'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab Puma workers not running (instance {{ $labels.instance }})
    description: "GitLab Puma on {{ $labels.instance }} has {{ $value }} running workers out of expected total.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 2GB may need adjustment based on your instance size.
  # High memory usage can lead to OOM kills and service disruptions.
- alert: GitlabHighMemoryUsage
  expr: 'process_resident_memory_bytes{job=~".*gitlab.*"} > 2e+9'
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: GitLab high memory usage (instance {{ $labels.instance }})
    description: "GitLab process on {{ $labels.instance }} is using {{ $value | humanize1024 }}B of RSS memory.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Heap fragmentation above 50% means a significant amount of memory is wasted.
  # A Puma worker restart may help reclaim memory.
- alert: GitlabRubyHeapFragmentation
  expr: 'ruby_gc_stat_ext_heap_fragmentation{job=~".*gitlab.*"} > 0.5'
  for: 15m
  labels:
    severity: warning
  annotations:
    summary: GitLab Ruby heap fragmentation (instance {{ $labels.instance }})
    description: "GitLab Ruby heap fragmentation on {{ $labels.instance }} is {{ $value }}. High fragmentation wastes memory.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Queued connections indicate Puma workers are saturated.
  # Consider increasing puma['worker_processes'] or puma['max_threads'] in gitlab.rb.
- alert: GitlabPumaHighQueuedConnections
  expr: 'puma_queued_connections > 5'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab Puma high queued connections (instance {{ $labels.instance }})
    description: "GitLab Puma has {{ $value }} queued connections on {{ $labels.instance }}. Requests are waiting for an available worker thread.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # When the pool is near saturation, requests may block waiting for a connection.
  # Increase db_pool_size in gitlab.rb or investigate slow queries.
- alert: GitlabDatabaseConnectionPoolSaturation
  expr: 'gitlab_database_connection_pool_busy / gitlab_database_connection_pool_size * 100 > 90 and gitlab_database_connection_pool_size > 0'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab database connection pool saturation (instance {{ $labels.instance }})
    description: "GitLab database connection pool on {{ $labels.instance }} ({{ $labels.class }}) is {{ $value }}% busy.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: GitlabDatabaseConnectionPoolDeadConnections
  expr: 'gitlab_database_connection_pool_dead > 0'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab database connection pool dead connections (instance {{ $labels.instance }})
    description: "GitLab database connection pool on {{ $labels.instance }} ({{ $labels.class }}) has {{ $value }} dead connections.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: GitlabDatabaseConnectionPoolWaiting
  expr: 'gitlab_database_connection_pool_waiting > 0'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab database connection pool waiting (instance {{ $labels.instance }})
    description: "GitLab on {{ $labels.instance }} has {{ $value }} threads waiting for a database connection.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: GitlabHighFileDescriptorUsage
  expr: 'process_open_fds{job=~".*gitlab.*"} / process_max_fds * 100 > 80 and process_max_fds > 0'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab high file descriptor usage (instance {{ $labels.instance }})
    description: "GitLab on {{ $labels.instance }} is using {{ $value }}% of available file descriptors.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: GitlabPumaHighThreadPoolUtilization
  expr: 'puma_active_connections / puma_max_threads * 100 > 90 and puma_max_threads > 0'
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: GitLab Puma high thread pool utilization (instance {{ $labels.instance }})
    description: "GitLab Puma thread pool on {{ $labels.instance }} is {{ $value }}% utilized, indicating sustained saturation before it becomes fully blocked.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: GitlabPumaNoAvailablePoolCapacity
  expr: 'puma_pool_capacity == 0'
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: GitLab Puma no available pool capacity (instance {{ $labels.instance }})
    description: "GitLab Puma pool capacity on {{ $labels.instance }} has been at 0 for 5 minutes. All threads are busy.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold is 5% of all requests returning server errors.
  # Check GitLab logs at /var/log/gitlab/ for root cause.
- alert: GitlabHighHttpErrorRate
  expr: 'sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) * 100 > 5 and sum(rate(http_requests_total[5m])) > 0'
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: GitLab high HTTP error rate (instance {{ $labels.instance }})
    description: "GitLab is returning more than 5% HTTP 5xx errors on {{ $labels.instance }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 10s may need adjustment based on your instance size and workload.
- alert: GitlabHighHttpRequestLatency
  expr: 'histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le)) > 10'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab high HTTP request latency (instance {{ $labels.instance }})
    description: "GitLab p95 HTTP request latency on {{ $labels.instance }} is above 10 seconds.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # This metric requires the emit_sidekiq_histogram_metrics feature flag to be enabled.
  # A sustained failure rate indicates background processing issues.
- alert: GitlabSidekiqJobsFailing
  expr: 'rate(sidekiq_jobs_failed_total[5m]) > 0.1'
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: GitLab Sidekiq jobs failing (instance {{ $labels.instance }})
    description: "GitLab Sidekiq jobs are failing at a rate of {{ $value }} per second on {{ $labels.instance }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # When running jobs approach the concurrency limit, new jobs will queue up.
  # Consider scaling Sidekiq workers or increasing concurrency.
- alert: GitlabSidekiqQueueTooLarge
  expr: 'sum(sidekiq_running_jobs) >= sum(sidekiq_concurrency) * 0.9'
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: GitLab Sidekiq queue too large (instance {{ $labels.instance }})
    description: "GitLab Sidekiq has {{ $value }} running jobs, approaching concurrency limit on {{ $labels.instance }}.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # This metric requires the emit_sidekiq_histogram_metrics feature flag to be enabled.
  # GitLab's largest finite completion bucket is 300 seconds, so p95 cannot exceed it.
  # This rule measures the percentage of completed jobs above that boundary, not an interpolated p95.
- alert: GitlabSidekiqHighJobCompletionTime
  expr: '100 * (sum by (worker) (rate(sidekiq_jobs_completion_seconds_bucket{le="+Inf"}[5m])) - sum by (worker) (rate(sidekiq_jobs_completion_seconds_bucket{le="300"}[5m]))) / sum by (worker) (rate(sidekiq_jobs_completion_seconds_bucket{le="+Inf"}[5m])) > 5'
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: GitLab Sidekiq high job completion time (instance {{ $labels.instance }})
    description: "More than 5% of observed GitLab Sidekiq jobs take longer than 300 seconds ({{ $value }}%).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # This metric requires the emit_sidekiq_histogram_metrics feature flag to be enabled.
  # High queue latency means jobs are stuck waiting. Check Sidekiq concurrency and queue sizes.
  # GitLab's largest finite queue-duration bucket is 60 seconds, so p95 cannot exceed it.
  # This rule measures the percentage of observed waits above that boundary, not an interpolated p95.
- alert: GitlabSidekiqHighQueueLatency
  expr: '100 * (sum(rate(sidekiq_jobs_queue_duration_seconds_bucket{le="+Inf"}[5m])) - sum(rate(sidekiq_jobs_queue_duration_seconds_bucket{le="60"}[5m]))) / sum(rate(sidekiq_jobs_queue_duration_seconds_bucket{le="+Inf"}[5m])) > 5'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab Sidekiq high queue latency (instance {{ $labels.instance }})
    description: "More than 5% of observed GitLab Sidekiq queue waits take longer than 60 seconds ({{ $value }}%).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # 30s threshold is a rough default; depends on your pipeline complexity and CI instance load — adjust based on your baseline.
- alert: GitlabCiPipelineCreationSlow
  expr: 'histogram_quantile(0.95, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[5m])) by (le)) > 30'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab CI pipeline creation slow (instance {{ $labels.instance }})
    description: "GitLab CI pipeline creation p95 latency on {{ $labels.instance }} is above 30 seconds.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # This metric may not exist in all GitLab versions. Verify against your GitLab installation.
- alert: GitlabCiPipelineFailuresIncreasing
  expr: 'deriv(gitlab_ci_pipeline_failure_reasons[5m]) > 0.05'
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: GitLab CI pipeline failures increasing (instance {{ $labels.instance }})
    description: "GitLab CI pipeline failures are increasing on {{ $labels.instance }} ({{ $value }}/s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: GitlabRackUncaughtErrors
  expr: 'rate(rack_uncaught_errors_total[5m]) > 0.05'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab rack uncaught errors (instance {{ $labels.instance }})
    description: "GitLab is experiencing uncaught errors in the Rack layer on {{ $labels.instance }} ({{ $value }}/s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # This may happen during a rolling deployment. If it persists, investigate incomplete upgrades.
- alert: GitlabVersionMismatch
  expr: 'count(count by (version) (gitlab_build_info)) > 1'
  for: 0m
  labels:
    severity: warning
  annotations:
    summary: GitLab version mismatch (instance {{ $labels.instance }})
    description: "Multiple GitLab versions are running across the fleet.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: GitlabRubyThreadsSaturated
  expr: 'sum by (instance) (gitlab_ruby_threads_running_threads) > on(instance) gitlab_ruby_threads_max_expected_threads * 1.5'
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: GitLab Ruby threads saturated (instance {{ $labels.instance }})
    description: "GitLab running threads on {{ $labels.instance }} have exceeded the expected maximum ({{ $value }}).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: GitlabSidekiqQueueNotDraining
  expr: 'sum by (name) (sidekiq_queue_size) > 0'
  for: 30m
  labels:
    severity: warning
  annotations:
    summary: GitLab Sidekiq queue not draining (instance {{ $labels.instance }})
    description: "GitLab Sidekiq queue {{ $labels.name }} has had a persistent backlog of {{ $value }} jobs for 30 minutes, suggesting a stuck or crashed worker class rather than transient load.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Frequent runner auth failures may indicate expired tokens or misconfigured runners.
- alert: GitlabCiRunnerAuthenticationFailures
  expr: 'increase(gitlab_ci_runner_authentication_failure_total[5m]) > 5'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab CI runner authentication failures (instance {{ $labels.instance }})
    description: "GitLab CI runners are experiencing authentication failures on {{ $labels.instance }} ({{ $value }} failures).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

groups:
- name: Workhorse
  rules:
      # Workhorse sits in front of Puma and handles Git HTTP, file uploads, and proxying.
      # Threshold from GitLab Omnibus default rules: 10% for high-traffic instances.
    - alert: GitlabWorkhorseHighErrorRate
      expr: 'sum(rate(gitlab_workhorse_http_request_duration_seconds_count{code=~"5.."}[5m])) / sum(rate(gitlab_workhorse_http_request_duration_seconds_count[5m])) * 100 > 10 and sum(rate(gitlab_workhorse_http_request_duration_seconds_count[5m])) > 0'
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: GitLab Workhorse high error rate (instance {{ $labels.instance }})
        description: "GitLab Workhorse on {{ $labels.instance }} is returning more than 10% HTTP 5xx errors.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # 10s threshold is a rough default; depends on your request mix (large file uploads/downloads vs typical Git operations) — adjust based on your baseline.
    - alert: GitlabWorkhorseHighLatency
      expr: 'histogram_quantile(0.95, sum(rate(gitlab_workhorse_http_request_duration_seconds_bucket[5m])) by (le)) > 10'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab Workhorse high latency (instance {{ $labels.instance }})
        description: "GitLab Workhorse on {{ $labels.instance }} p95 request latency is above 10 seconds.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Threshold of 100 may need adjustment based on instance size.
    - alert: GitlabWorkhorseHighIn-flightRequests
      expr: 'gitlab_workhorse_http_in_flight_requests > 100'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab Workhorse high in-flight requests (instance {{ $labels.instance }})
        description: "GitLab Workhorse on {{ $labels.instance }} has {{ $value }} in-flight requests.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

wget https://raw.githubusercontent.com/samber/awesome-prometheus-alerts/refs/heads/master/dist/rules/gitlab-ci/workhorse.yml

  # Workhorse sits in front of Puma and handles Git HTTP, file uploads, and proxying.
  # Threshold from GitLab Omnibus default rules: 10% for high-traffic instances.
- alert: GitlabWorkhorseHighErrorRate
  expr: 'sum(rate(gitlab_workhorse_http_request_duration_seconds_count{code=~"5.."}[5m])) / sum(rate(gitlab_workhorse_http_request_duration_seconds_count[5m])) * 100 > 10 and sum(rate(gitlab_workhorse_http_request_duration_seconds_count[5m])) > 0'
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: GitLab Workhorse high error rate (instance {{ $labels.instance }})
    description: "GitLab Workhorse on {{ $labels.instance }} is returning more than 10% HTTP 5xx errors.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # 10s threshold is a rough default; depends on your request mix (large file uploads/downloads vs typical Git operations) — adjust based on your baseline.
- alert: GitlabWorkhorseHighLatency
  expr: 'histogram_quantile(0.95, sum(rate(gitlab_workhorse_http_request_duration_seconds_bucket[5m])) by (le)) > 10'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab Workhorse high latency (instance {{ $labels.instance }})
    description: "GitLab Workhorse on {{ $labels.instance }} p95 request latency is above 10 seconds.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Threshold of 100 may need adjustment based on instance size.
- alert: GitlabWorkhorseHighIn-flightRequests
  expr: 'gitlab_workhorse_http_in_flight_requests > 100'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab Workhorse high in-flight requests (instance {{ $labels.instance }})
    description: "GitLab Workhorse on {{ $labels.instance }} has {{ $value }} in-flight requests.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

groups:
- name: Gitaly
  rules:
      # Brief throttling spikes are normal. Threshold of 0.1s/s (10% of CPU time throttled) filters out transient noise.
    - alert: GitlabGitalyCpuThrottled
      expr: 'rate(gitaly_cgroup_cpu_cfs_throttled_seconds_total[5m]) > 0.1'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab Gitaly CPU throttled (instance {{ $labels.instance }})
        description: "Gitaly processes on {{ $labels.instance }} are being CPU throttled by cgroups.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # ResourceExhausted errors from Gitaly mean Git operations are being rejected due to
      # concurrency limits. This directly impacts users trying to push, pull, or clone.
    - alert: GitlabGitalyResourceExhausted
      expr: 'sum(rate(grpc_server_handled_total{job="gitaly",grpc_code="ResourceExhausted"}[5m])) / sum(rate(grpc_server_handled_total{job="gitaly"}[5m])) * 100 > 1 and sum(rate(grpc_server_handled_total{job="gitaly"}[5m])) > 0'
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: GitLab Gitaly resource exhausted (instance {{ $labels.instance }})
        description: "Gitaly on {{ $labels.instance }} is returning ResourceExhausted errors, indicating overload ({{ $value }}%).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # When the circuit breaker trips to "open" state, Git operations (push, pull, clone) will fail.
      # Check Gitaly service health and logs.
    - alert: GitlabGitalyCircuitBreakerTripped
      expr: 'increase(gitaly_circuit_breaker_transitions_total{to_state="open"}[5m]) > 0'
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: GitLab Gitaly circuit breaker tripped (instance {{ $labels.instance }})
        description: "Gitaly circuit breaker has tripped on {{ $labels.instance }}. Git operations are failing.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Counts server-side failures only. NotFound, AlreadyExists and Cancelled are normal gRPC
      # responses and are deliberately excluded from the error rate.
    - alert: GitlabGitalyHighGrpcErrorRate
      expr: 'sum(rate(grpc_server_handled_total{job="gitaly",grpc_code=~"Internal|Unavailable|DeadlineExceeded|ResourceExhausted|Aborted|Unknown|DataLoss"}[5m])) / sum(rate(grpc_server_handled_total{job="gitaly"}[5m])) * 100 > 5 and sum(rate(grpc_server_handled_total{job="gitaly"}[5m])) > 0'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab Gitaly high gRPC error rate (instance {{ $labels.instance }})
        description: "Gitaly on {{ $labels.instance }} is returning more than 5% gRPC errors.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # Replace job="gitaly" with the actual job name in your Prometheus configuration. The 1s p95 latency threshold is a rough default; adjust based on your workload.
    - alert: GitlabGitalyHighRpcLatency
      expr: 'histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{job="gitaly",grpc_type="unary"}[5m])) by (le)) > 1'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: GitLab Gitaly high RPC latency (instance {{ $labels.instance }})
        description: "Gitaly on {{ $labels.instance }} p95 unary RPC latency exceeds 1 second ({{ $value }}s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: GitlabGitalyAuthenticationFailures
      expr: 'increase(gitaly_authentications_total{status="denied"}[5m]) > 3'
      for: 0m
      labels:
        severity: warning
      annotations:
        summary: GitLab Gitaly authentication failures (instance {{ $labels.instance }})
        description: "Gitaly on {{ $labels.instance }} has authentication failures ({{ $value }}).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

wget https://raw.githubusercontent.com/samber/awesome-prometheus-alerts/refs/heads/master/dist/rules/gitlab-ci/gitaly.yml

  # Brief throttling spikes are normal. Threshold of 0.1s/s (10% of CPU time throttled) filters out transient noise.
- alert: GitlabGitalyCpuThrottled
  expr: 'rate(gitaly_cgroup_cpu_cfs_throttled_seconds_total[5m]) > 0.1'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab Gitaly CPU throttled (instance {{ $labels.instance }})
    description: "Gitaly processes on {{ $labels.instance }} are being CPU throttled by cgroups.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # ResourceExhausted errors from Gitaly mean Git operations are being rejected due to
  # concurrency limits. This directly impacts users trying to push, pull, or clone.
- alert: GitlabGitalyResourceExhausted
  expr: 'sum(rate(grpc_server_handled_total{job="gitaly",grpc_code="ResourceExhausted"}[5m])) / sum(rate(grpc_server_handled_total{job="gitaly"}[5m])) * 100 > 1 and sum(rate(grpc_server_handled_total{job="gitaly"}[5m])) > 0'
  for: 5m
  labels:
    severity: critical
  annotations:
    summary: GitLab Gitaly resource exhausted (instance {{ $labels.instance }})
    description: "Gitaly on {{ $labels.instance }} is returning ResourceExhausted errors, indicating overload ({{ $value }}%).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # When the circuit breaker trips to "open" state, Git operations (push, pull, clone) will fail.
  # Check Gitaly service health and logs.
- alert: GitlabGitalyCircuitBreakerTripped
  expr: 'increase(gitaly_circuit_breaker_transitions_total{to_state="open"}[5m]) > 0'
  for: 0m
  labels:
    severity: critical
  annotations:
    summary: GitLab Gitaly circuit breaker tripped (instance {{ $labels.instance }})
    description: "Gitaly circuit breaker has tripped on {{ $labels.instance }}. Git operations are failing.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Counts server-side failures only. NotFound, AlreadyExists and Cancelled are normal gRPC
  # responses and are deliberately excluded from the error rate.
- alert: GitlabGitalyHighGrpcErrorRate
  expr: 'sum(rate(grpc_server_handled_total{job="gitaly",grpc_code=~"Internal|Unavailable|DeadlineExceeded|ResourceExhausted|Aborted|Unknown|DataLoss"}[5m])) / sum(rate(grpc_server_handled_total{job="gitaly"}[5m])) * 100 > 5 and sum(rate(grpc_server_handled_total{job="gitaly"}[5m])) > 0'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab Gitaly high gRPC error rate (instance {{ $labels.instance }})
    description: "Gitaly on {{ $labels.instance }} is returning more than 5% gRPC errors.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # Replace job="gitaly" with the actual job name in your Prometheus configuration. The 1s p95 latency threshold is a rough default; adjust based on your workload.
- alert: GitlabGitalyHighRpcLatency
  expr: 'histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{job="gitaly",grpc_type="unary"}[5m])) by (le)) > 1'
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: GitLab Gitaly high RPC latency (instance {{ $labels.instance }})
    description: "Gitaly on {{ $labels.instance }} p95 unary RPC latency exceeds 1 second ({{ $value }}s).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: GitlabGitalyAuthenticationFailures
  expr: 'increase(gitaly_authentications_total{status="denied"}[5m]) > 3'
  for: 0m
  labels:
    severity: warning
  annotations:
    summary: GitLab Gitaly authentication failures (instance {{ $labels.instance }})
    description: "Gitaly on {{ $labels.instance }} has authentication failures ({{ $value }}).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"