{% raw %}
groups:
  - name: demo
    rules:
      - alert: Caddy Down
        expr: up{job="caddy"} == 0
        for: 1m
        labels:
          severity: critical
      
      # 5% 4xx rate is a rough default. Client-error rates vary widely by application (bots, API misuse, rate limits, short-lived token expiry, malformed clients) — adjust based on your baseline.
      - alert: CaddyHighHTTP4xxErrorRateService
        expr: sum(rate(caddy_http_request_duration_seconds_count{code=~"4.."}[3m])) by (instance) / sum(rate(caddy_http_request_duration_seconds_count[3m])) by (instance) * 100 > 5 and sum(rate(caddy_http_request_duration_seconds_count[3m])) by (instance) > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: Caddy high HTTP 4xx error rate service (instance {{ $labels.instance }})
          description: "Caddy service 4xx error rate is above 5%\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
{% endraw %}