Prometheus and Grafana — Production Monitoring Stack Setup

# Docker Compose stack
cat > docker-compose.yml << 'EOF'
version: '3.8'
 
services:
  prometheus:
    image: prom/prometheus:v2.48.0
    container_name: prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./rules/:/etc/prometheus/rules/
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
      - '--web.enable-lifecycle'
      - '--web.enable-admin-api'
    restart: unless-stopped
 
  grafana:
    image: grafana/grafana:10.2.0
    container_name: grafana
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=secure-password
      - GF_USERS_ALLOW_SIGN_UP=false
    restart: unless-stopped
 
  alertmanager:
    image: prom/alertmanager:v0.26.0
    container_name: alertmanager
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml
 
volumes:
  prometheus_data:
  grafana_data:
EOF
 
docker compose up -d

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s
  external_labels:
    cluster: production
    region: us-east-1
 
rule_files:
  - "rules/*.yml"
 
alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']
 
scrape_configs:
  # Prometheus self-monitoring
  - job_name: prometheus
    static_configs:
      - targets: ['localhost:9090']
 
  # Node exporter — OS metrics
  - job_name: node
    static_configs:
      - targets:
          - web1.example.com:9100
          - web2.example.com:9100
          - db1.example.com:9100
 
  # Application metrics
  - job_name: my-app
    metrics_path: /metrics
    static_configs:
      - targets: ['app:3000']
 
  # Kubernetes pods (auto-discovery)
  - job_name: kubernetes-pods
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)

// Node.js with prom-client
const promClient = require('prom-client');
const express = require('express');
const app = express();
 
// Collect default metrics (CPU, memory, event loop)
promClient.collectDefaultMetrics({ prefix: 'myapp_' });
 
// Custom counters
const httpRequestsTotal = new promClient.Counter({
  name: 'myapp_http_requests_total',
  help: 'Total HTTP requests',
  labelNames: ['method', 'route', 'status_code']
});
 
// Custom histograms (latency)
const httpDuration = new promClient.Histogram({
  name: 'myapp_http_request_duration_seconds',
  help: 'HTTP request duration',
  labelNames: ['method', 'route'],
  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]
});
 
// Middleware
app.use((req, res, next) => {
  const end = httpDuration.startTimer({ method: req.method, route: req.path });
  res.on('finish', () => {
    httpRequestsTotal.inc({ method: req.method, route: req.path, status_code: res.statusCode });
    end();
  });
  next();
});
 
// Expose metrics endpoint
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', promClient.register.contentType);
  res.end(await promClient.register.metrics());
});

# Request rate (requests per second, 5-minute window)
rate(myapp_http_requests_total[5m])
 
# Error rate (percentage)
sum(rate(myapp_http_requests_total{status_code=~"5.."}[5m]))
/
sum(rate(myapp_http_requests_total[5m])) * 100
 
# 95th percentile latency
histogram_quantile(0.95, rate(myapp_http_request_duration_seconds_bucket[5m]))
 
# CPU usage per node
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
 
# Memory usage
node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100
 
# Disk usage
(node_filesystem_size_bytes - node_filesystem_free_bytes) / node_filesystem_size_bytes * 100

# rules/application.yml
groups:
  - name: application
    rules:
      - alert: HighErrorRate
        expr: |
          sum(rate(myapp_http_requests_total{status_code=~"5.."}[5m]))
          /
          sum(rate(myapp_http_requests_total[5m])) > 0.05
        for: 2m
        labels:
          severity: critical
          team: backend
        annotations:
          summary: "High error rate detected"
          description: "Error rate is {{ $value | humanizePercentage }} (threshold 5%)"
          runbook: "https://wiki.example.com/runbooks/high-error-rate"
 
      - alert: HighLatency
        expr: |
          histogram_quantile(0.95,
            rate(myapp_http_request_duration_seconds_bucket[5m])
          ) > 1.0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "P95 latency above 1 second"
 
      - alert: InstanceDown
        expr: up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Instance {{ $labels.instance }} is down"

# alertmanager.yml
global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/...'
 
route:
  group_by: ['alertname', 'cluster']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: default-receiver
 
  routes:
    - match:
        severity: critical
      receiver: pagerduty-critical
      continue: true
    - match:
        team: backend
      receiver: backend-slack
 
receivers:
  - name: default-receiver
    slack_configs:
      - channel: '#alerts'
        text: '{{ template "slack.default.text" . }}'
 
  - name: pagerduty-critical
    pagerduty_configs:
      - routing_key: YOUR_PAGERDUTY_KEY
 
  - name: backend-slack
    slack_configs:
      - channel: '#backend-alerts'