Build your own monitoring server with Docker, Prometheus and Grafana

# Copy to .env and change the password before the first start.
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=change-me-now
PROM_RETENTION=30d
PROM_PORT=9090
GRAFANA_PORT=3000
ALERTMANAGER_PORT=9093
PUSHGATEWAY_PORT=9091

# Monitoring server: Prometheus + Grafana + Alertmanager + exporters.
# Start:  docker compose up -d     Stop: docker compose down
name: monitoring

x-defaults: &defaults
  restart: unless-stopped
  networks: [monitoring]
  logging:
    driver: json-file
    options: { max-size: "10m", max-file: "3" }

services:
  prometheus:
    <<: *defaults
    image: prom/prometheus:v3.15.1
    container_name: prometheus
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=${PROM_RETENTION:-30d}
      - --web.enable-lifecycle          # allows: curl -X POST localhost:9090/-/reload
    volumes:
      - ./prometheus:/etc/prometheus:ro
      - prometheus-data:/prometheus
    ports:
      - "${PROM_PORT:-9090}:9090"
    extra_hosts:
      - "host.docker.internal:host-gateway"   # lets Prometheus scrape apps running on the host

  alertmanager:
    <<: *defaults
    image: prom/alertmanager:v0.34.2
    container_name: alertmanager
    command:
      - --config.file=/etc/alertmanager/alertmanager.yml
      - --storage.path=/alertmanager
    volumes:
      - ./alertmanager:/etc/alertmanager:ro
      - alertmanager-data:/alertmanager
    ports:
      - "${ALERTMANAGER_PORT:-9093}:9093"

  grafana:
    <<: *defaults
    image: grafana/grafana:13.3.0
    container_name: grafana
    environment:
      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env}
      GF_USERS_ALLOW_SIGN_UP: "false"
      GF_ANALYTICS_REPORTING_ENABLED: "false"
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
      - grafana-data:/var/lib/grafana
    ports:
      - "${GRAFANA_PORT:-3000}:3000"
    depends_on: [prometheus]

  node-exporter:
    <<: *defaults
    image: prom/node-exporter:v1.12.1
    container_name: node-exporter
    command:
      - --path.rootfs=/host
      - --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)
    pid: host
    volumes:
      - /:/host:ro,rslave

  cadvisor:
    <<: *defaults
    image: ghcr.io/google/cadvisor:v0.60.6
    container_name: cadvisor
    privileged: true
    devices: ["/dev/kmsg"]
    command:
      - --docker_only=true
      - --housekeeping_interval=15s
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro

  blackbox:
    <<: *defaults
    image: prom/blackbox-exporter:v0.29.1
    container_name: blackbox
    command: [--config.file=/etc/blackbox/blackbox.yml]
    volumes:
      - ./blackbox:/etc/blackbox:ro

  pushgateway:
    <<: *defaults
    image: prom/pushgateway:v1.11.5
    container_name: pushgateway
    ports:
      - "${PUSHGATEWAY_PORT:-9091}:9091"

networks:
  monitoring:

volumes:
  prometheus-data:
  alertmanager-data:
  grafana-data:

global:
  scrape_interval: 15s        # how often to pull metrics
  evaluation_interval: 15s    # how often to evaluate alert rules
  external_labels:
    monitor: lab-monitoring

rule_files:
  - /etc/prometheus/rules/*.yml

alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

scrape_configs:
  # Prometheus watching itself
  - job_name: prometheus
    static_configs:
      - targets: ["localhost:9090"]

  # This server's CPU, memory, disk, network
  - job_name: node
    static_configs:
      - targets: ["node-exporter:9100"]
        labels: { instance_name: monitoring-server }
    # Other machines: list them in targets/nodes.yml (reloaded automatically)
    file_sd_configs:
      - files: ["/etc/prometheus/targets/nodes.yml"]
        refresh_interval: 1m

  # Per-container CPU, memory, network, restarts
  - job_name: cadvisor
    static_configs:
      - targets: ["cadvisor:8080"]

  # Is this website up, and how fast does it answer?
  - job_name: blackbox-http
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
          - https://prometheus.io
          - http://grafana:3000/api/health
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox:9115

  # Metrics pushed by short batch jobs
  - job_name: pushgateway
    honor_labels: true          # keep the job/instance labels the batch job pushed
    static_configs:
      - targets: ["pushgateway:9091"]

  # Long-running experiments that expose /metrics (see experiment/train_with_metrics.py)
  - job_name: experiments
    scrape_interval: 5s
    static_configs:
      - targets: ["host.docker.internal:8000"]
        labels: { experiment: demo-training }

  - job_name: alertmanager
    static_configs:
      - targets: ["alertmanager:9093"]

# Add one entry per machine running node-exporter on port 9100.
# Prometheus re-reads this file every minute; no restart needed.
# - targets: ["192.168.1.21:9100"]
#   labels: { instance_name: jetson-01, role: edge }
# - targets: ["192.168.1.22:9100"]
#   labels: { instance_name: rpi5-01, role: edge }
[]

groups:
  - name: hosts
    rules:
      - alert: TargetDown
        expr: up == 0
        for: 2m
        labels: { severity: critical }
        annotations:
          summary: "{{ $labels.job }} target {{ $labels.instance }} is down"
          description: "Prometheus could not scrape it for 2 minutes."

      - alert: HighCpuUsage
        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
        for: 10m
        labels: { severity: warning }
        annotations:
          summary: "CPU above 90% on {{ $labels.instance }}"
          description: "Current value: {{ $value | printf \"%.1f\" }}%"

      - alert: LowMemory
        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.10
        for: 5m
        labels: { severity: warning }
        annotations:
          summary: "Less than 10% memory left on {{ $labels.instance }}"

      - alert: DiskWillFillIn24h
        expr: predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}[6h], 24 * 3600) < 0
        for: 30m
        labels: { severity: warning }
        annotations:
          summary: "{{ $labels.mountpoint }} on {{ $labels.instance }} will be full within 24 h"

  - name: services
    rules:
      - alert: WebsiteDown
        expr: probe_success{job="blackbox-http"} == 0
        for: 2m
        labels: { severity: critical }
        annotations:
          summary: "{{ $labels.instance }} is not responding"

      - alert: ContainerRestarting
        expr: changes(container_start_time_seconds{name!=""}[15m]) > 2
        labels: { severity: warning }
        annotations:
          summary: "Container {{ $labels.name }} restarted more than twice in 15 minutes"

  - name: experiments
    rules:
      - alert: ExperimentStalled
        expr: time() - experiment_last_step_timestamp_seconds > 600
        for: 1m
        labels: { severity: warning }
        annotations:
          summary: "Experiment {{ $labels.experiment }} has not reported progress for 10 minutes"

      - alert: BatchJobFailed
        expr: batch_job_success == 0
        labels: { severity: critical }
        annotations:
          summary: "Batch job {{ $labels.job }} reported failure"

route:
  receiver: default
  group_by: [alertname, instance]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - matchers: [severity="critical"]
      receiver: default
      repeat_interval: 1h

receivers:
  - name: default
    # Uncomment ONE of these and fill in your details.
    # email_configs:
    #   - to: you@example.com
    #     from: alerts@example.com
    #     smarthost: smtp.gmail.com:587
    #     auth_username: you@example.com
    #     auth_password: your-app-password
    # telegram_configs:
    #   - bot_token: "123456:ABC..."
    #     chat_id: 123456789
    # slack_configs:
    #   - api_url: https://hooks.slack.com/services/XXX/YYY/ZZZ
    #     channel: "#alerts"

inhibit_rules:
  - source_matchers: [severity="critical"]
    target_matchers: [severity="warning"]
    equal: [instance]

modules:
  http_2xx:
    prober: http
    timeout: 10s
    http:
      preferred_ip_protocol: ip4
      valid_status_codes: []   # empty = any 2xx
      follow_redirects: true
  tcp_connect:
    prober: tcp
    timeout: 5s
  icmp:
    prober: icmp
    timeout: 5s

apiVersion: 1
datasources:
  - name: Prometheus
    uid: prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    jsonData:
      timeInterval: 15s

apiVersion: 1
providers:
  - name: lab
    folder: Lab
    type: file
    allowUiUpdates: true
    options:
      path: /var/lib/grafana/dashboards

"""Expose live metrics from a long-running experiment so Prometheus can scrape them.

    pip install prometheus-client
    python train_with_metrics.py          # metrics at http://localhost:8000/metrics

Replace the fake training loop with your own code; keep the metric updates.
"""
import math
import random
import time

from prometheus_client import Counter, Gauge, Histogram, start_http_server

LOSS = Gauge("experiment_loss", "Current training loss")
ACCURACY = Gauge("experiment_accuracy", "Current validation accuracy (0-1)")
EPOCH = Gauge("experiment_epoch", "Current epoch")
LR = Gauge("experiment_learning_rate", "Current learning rate")
STEPS = Counter("experiment_steps", "Training steps completed")  # exported as experiment_steps_total
STEP_TIME = Histogram(
    "experiment_step_duration_seconds",
    "Time per training step",
    buckets=(0.05, 0.1, 0.2, 0.3, 0.5, 1, 2),
)
LAST_STEP = Gauge("experiment_last_step_timestamp_seconds", "Unix time of the last completed step")


def train(epochs: int = 50, steps_per_epoch: int = 100) -> None:
    lr = 0.01
    for epoch in range(epochs):
        EPOCH.set(epoch)
        LR.set(lr)
        for step in range(steps_per_epoch):
            with STEP_TIME.time():                      # measures how long this block takes
                time.sleep(random.uniform(0.08, 0.25))  # <- your real training step goes here
            progress = epoch + step / steps_per_epoch
            LOSS.set(2.3 * math.exp(-progress / 8) + random.uniform(0, 0.05))
            STEPS.inc()
            LAST_STEP.set_to_current_time()
        ACCURACY.set(min(0.99, 1 - math.exp(-(epoch + 1) / 6) + random.uniform(-0.01, 0.01)))
        lr *= 0.95


if __name__ == "__main__":
    start_http_server(8000)   # serves /metrics in a background thread
    train()

#!/usr/bin/env bash
# Run any command and push its outcome to the Pushgateway.
# Usage: ./run_batch_job.sh <job-name> <command...>
# Example: ./run_batch_job.sh nightly-backup tar czf /backup/data.tgz /data
set -uo pipefail
PUSHGATEWAY="${PUSHGATEWAY:-http://localhost:9091}"
JOB="$1"; shift

start=$(date +%s.%N)
"$@"; status=$?
end=$(date +%s.%N)
duration=$(echo "$end - $start" | bc)
success=$([ "$status" -eq 0 ] && echo 1 || echo 0)

cat <<METRICS | curl -s --data-binary @- "$PUSHGATEWAY/metrics/job/$JOB/instance/$(hostname)"
# TYPE batch_job_duration_seconds gauge
batch_job_duration_seconds $duration
# TYPE batch_job_success gauge
batch_job_success $success
# TYPE batch_job_last_run_timestamp_seconds gauge
batch_job_last_run_timestamp_seconds $(date +%s)
METRICS

echo "job=$JOB status=$status duration=${duration}s pushed to $PUSHGATEWAY"
exit "$status"

"""Push final results of a short experiment (one that ends before Prometheus can scrape it).

    pip install prometheus-client
    python push_results.py
"""
import time

from prometheus_client import CollectorRegistry, Gauge, push_to_gateway

registry = CollectorRegistry()
accuracy = Gauge("experiment_final_accuracy", "Accuracy at the end of the run", ["model"], registry=registry)
duration = Gauge("experiment_duration_seconds", "Wall-clock time of the run", ["model"], registry=registry)
success = Gauge("batch_job_success", "1 if the run finished without error", registry=registry)

start = time.time()
result = 0.947                       # <- run your experiment here and collect its result
accuracy.labels(model="xor-arbiter-lr").set(result)
duration.labels(model="xor-arbiter-lr").set(time.time() - start)
success.set(1)

push_to_gateway("localhost:9091", job="puf-attack-sweep", registry=registry)
print("pushed")