Nginx Prometheus Alert Rules

groups:
- name: KnyarNginxExporter
  rules:
      # 5% 4xx rate is a rough default. Client-error rates vary widely by application (bots, API misuse, rate limits, short-lived token expiry, scanners) — adjust based on your baseline.
    - alert: NginxHighHTTP4xxErrorRate
      expr: sum(rate(nginx_http_requests_total{status=~"^4.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5 and sum(rate(nginx_http_requests_total[1m])) > 0
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: Nginx high HTTP 4xx error rate (instance {{ $labels.instance }})
        description: "Too many HTTP requests with status 4xx (> 5%)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: NginxHighHTTP5xxErrorRate
      expr: sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5 and sum(rate(nginx_http_requests_total[1m])) > 0
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: Nginx high HTTP 5xx error rate (instance {{ $labels.instance }})
        description: "Too many HTTP requests with status 5xx (> 5%)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
      # 3s threshold is a rough default; acceptable latency varies by route and your SLO — adjust based on your baseline.
    - alert: NginxLatencyHigh
      expr: histogram_quantile(0.99, sum(rate(nginx_http_request_duration_seconds_bucket[2m])) by (host, node, le)) > 3
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: Nginx latency high (instance {{ $labels.instance }})
        description: "Nginx p99 latency is higher than 3 seconds\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
    
    - alert: NginxInternalMetricErrors
      expr: increase(nginx_metric_errors_total[5m]) > 0
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: Nginx internal metric errors (instance {{ $labels.instance }})
        description: "The nginx-lua-prometheus library failed to record one or more metrics (e.g. the shared dictionary used to store metrics is full or an LRU eviction occurred), which may mean other Nginx metrics are incomplete.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

wget https://raw.githubusercontent.com/samber/awesome-prometheus-alerts/refs/heads/master/dist/rules/nginx/knyar-nginx-exporter.yml

  # 5% 4xx rate is a rough default. Client-error rates vary widely by application (bots, API misuse, rate limits, short-lived token expiry, scanners) — adjust based on your baseline.
- alert: NginxHighHTTP4xxErrorRate
  expr: sum(rate(nginx_http_requests_total{status=~"^4.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5 and sum(rate(nginx_http_requests_total[1m])) > 0
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: Nginx high HTTP 4xx error rate (instance {{ $labels.instance }})
    description: "Too many HTTP requests with status 4xx (> 5%)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: NginxHighHTTP5xxErrorRate
  expr: sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5 and sum(rate(nginx_http_requests_total[1m])) > 0
  for: 1m
  labels:
    severity: critical
  annotations:
    summary: Nginx high HTTP 5xx error rate (instance {{ $labels.instance }})
    description: "Too many HTTP requests with status 5xx (> 5%)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  # 3s threshold is a rough default; acceptable latency varies by route and your SLO — adjust based on your baseline.
- alert: NginxLatencyHigh
  expr: histogram_quantile(0.99, sum(rate(nginx_http_request_duration_seconds_bucket[2m])) by (host, node, le)) > 3
  for: 2m
  labels:
    severity: warning
  annotations:
    summary: Nginx latency high (instance {{ $labels.instance }})
    description: "Nginx p99 latency is higher than 3 seconds\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: NginxInternalMetricErrors
  expr: increase(nginx_metric_errors_total[5m]) > 0
  for: 1m
  labels:
    severity: warning
  annotations:
    summary: Nginx internal metric errors (instance {{ $labels.instance }})
    description: "The nginx-lua-prometheus library failed to record one or more metrics (e.g. the shared dictionary used to store metrics is full or an LRU eviction occurred), which may mean other Nginx metrics are incomplete.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"