Monitoring and Observability

Comprehensive monitoring and observability provide visibility into application behavior. Learn to implement monitoring systems for production environments.

Challenge

Build a complete monitoring and observability system that provides insights into application performance, reliability, and user experience.

Explanation

Three Pillars of Observability

Metrics: Quantitative measurements

Throughput, latency, error rates
Resource utilization
Custom business metrics

Logs: Event records

Request logs
Error logs
Application logs

Traces: Request flow

Distributed tracing
Request path
Latency breakdown

Monitoring Strategy

Infrastructure: CPU, memory, disk
Application: Requests, errors, performance
Business: User actions, conversions
Alerts: Threshold violations

Code Examples

1. Prometheus Metrics

const prometheus = require('prom-client');
const express = require('express');

// Default metrics
prometheus.collectDefaultMetrics();

// Custom metrics
const httpRequestDuration = new prometheus.Histogram({
  name: 'http_request_duration_seconds',
  help: 'Duration of HTTP requests in seconds',
  labelNames: ['method', 'route', 'status_code'],
  buckets: [0.1, 0.5, 1, 2, 5, 10],
});

const httpRequestsTotal = new prometheus.Counter({
  name: 'http_requests_total',
  help: 'Total HTTP requests',
  labelNames: ['method', 'route', 'status_code'],
});

const activeConnections = new prometheus.Gauge({
  name: 'active_connections',
  help: 'Number of active connections',
});

const databaseConnections = new prometheus.Gauge({
  name: 'database_connections',
  help: 'Database pool connections',
  labelNames: ['pool_name', 'state'],
});

// Middleware
app.use((req, res, next) => {
  const start = Date.now();
  activeConnections.inc();

  res.on('finish', () => {
    const duration = (Date.now() - start) / 1000;
    httpRequestDuration
      .labels(req.method, req.route?.path || req.path, res.statusCode)
      .observe(duration);

    httpRequestsTotal
      .labels(req.method, req.route?.path || req.path, res.statusCode)
      .inc();

    activeConnections.dec();
  });

  next();
});

// Metrics endpoint
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', prometheus.register.contentType);
  res.end(await prometheus.register.metrics());
});

2. Distributed Tracing

const { NodeSDK } = require('@opentelemetry/sdk-node');
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');
const { JaegerExporter } = require('@opentelemetry/exporter-jaeger-compact');
const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base');
const { ConsoleSpanExporter, SimpleSpanProcessor } = require('@opentelemetry/sdk-trace-base');

// Initialize Jaeger exporter
const jaegerExporter = new JaegerExporter({
  host: process.env.JAEGER_HOST || 'localhost',
  port: parseInt(process.env.JAEGER_PORT || '6831'),
});

// Initialize SDK
const sdk = new NodeSDK({
  traceExporter: jaegerExporter,
  instrumentations: [getNodeAutoInstrumentations()],
});

sdk.start();
console.log('Tracing initialized');

// Usage in Express
const { context, trace } = require('@opentelemetry/api');

app.get('/api/users/:id', async (req, res) => {
  const span = trace.getActiveSpan();
  span?.setAttribute('user.id', req.params.id);

  const userSpan = tracer.startSpan('fetch-user');
  try {
    const user = await fetchUser(req.params.id);
    userSpan.addEvent('user-fetched', { userId: user.id });
    res.json(user);
  } catch (error) {
    userSpan.recordException(error);
    res.status(500).json({ error: error.message });
  } finally {
    userSpan.end();
  }
});

3. Custom Observability Dashboard

class ObservabilityCollector {
  constructor() {
    this.metrics = new Map();
    this.alerts = [];
    this.thresholds = new Map();
  }

  recordMetric(name, value, labels = {}) {
    const key = `${name}:${JSON.stringify(labels)}`;

    if (!this.metrics.has(name)) {
      this.metrics.set(name, []);
    }

    this.metrics.get(name).push({
      value,
      labels,
      timestamp: Date.now(),
    });

    this.checkThresholds(name, value, labels);
  }

  setThreshold(name, condition, alertMessage) {
    if (!this.thresholds.has(name)) {
      this.thresholds.set(name, []);
    }
    this.thresholds.get(name).push({ condition, alertMessage });
  }

  checkThresholds(name, value, labels) {
    const conditions = this.thresholds.get(name) || [];

    conditions.forEach(({ condition, alertMessage }) => {
      if (condition(value)) {
        this.alerts.push({
          timestamp: Date.now(),
          metric: name,
          value,
          labels,
          message: alertMessage,
        });
      }
    });
  }

  getMetrics(name, timeWindow = 60000) {
    const now = Date.now();
    const values = this.metrics.get(name) || [];

    return values
      .filter(m => now - m.timestamp < timeWindow)
      .map(m => ({
        value: m.value,
        labels: m.labels,
        time: new Date(m.timestamp),
      }));
  }

  getStats(name) {
    const values = this.metrics.get(name) || [];
    const numericValues = values.map(v => v.value);

    if (numericValues.length === 0) {
      return null;
    }

    const sorted = numericValues.sort((a, b) => a - b);

    return {
      count: numericValues.length,
      avg: numericValues.reduce((a, b) => a + b) / numericValues.length,
      min: sorted[0],
      max: sorted[numericValues.length - 1],
      p50: sorted[Math.floor(numericValues.length * 0.5)],
      p95: sorted[Math.floor(numericValues.length * 0.95)],
      p99: sorted[Math.floor(numericValues.length * 0.99)],
    };
  }

  getAlerts(limit = 100) {
    return this.alerts.slice(-limit);
  }
}

// Setup
const collector = new ObservabilityCollector();

collector.setThreshold('response-time',
  (value) => value > 1000,
  'Response time exceeded 1 second'
);

collector.setThreshold('error-rate',
  (value) => value > 0.05,
  'Error rate exceeded 5%'
);

// Middleware to collect metrics
app.use((req, res, next) => {
  const start = Date.now();

  res.on('finish', () => {
    const duration = Date.now() - start;
    collector.recordMetric('response-time', duration, {
      method: req.method,
      path: req.path,
    });

    const isError = res.statusCode >= 400 ? 1 : 0;
    collector.recordMetric('error-rate', isError, {
      method: req.method,
    });
  });

  next();
});

// Observability endpoint
app.get('/observability/dashboard', (req, res) => {
  res.json({
    metrics: {
      responseTime: collector.getStats('response-time'),
      errorRate: collector.getStats('error-rate'),
    },
    recentAlerts: collector.getAlerts(10),
  });
});

4. Health Metrics Aggregation

class HealthMetricsAggregator {
  constructor() {
    this.metrics = {
      uptime: 0,
      errorCount: 0,
      requestCount: 0,
      avgResponseTime: 0,
      dbConnections: { total: 0, available: 0 },
      memoryUsage: { heapUsed: 0, heapTotal: 0 },
      cpuUsage: 0,
    };

    this.startTime = Date.now();
  }

  collectMetrics() {
    const memUsage = process.memoryUsage();
    const uptime = Math.floor((Date.now() - this.startTime) / 1000);

    return {
      uptime,
      timestamp: new Date().toISOString(),
      process: {
        memory: {
          heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024),
          heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024),
          external: Math.round(memUsage.external / 1024 / 1024),
        },
        cpuUsage: process.cpuUsage(),
      },
      os: {
        platform: process.platform,
        arch: process.arch,
        freeMemory: Math.round(require('os').freemem() / 1024 / 1024),
        totalMemory: Math.round(require('os').totalmem() / 1024 / 1024),
      },
      requests: {
        total: this.metrics.requestCount,
        errors: this.metrics.errorCount,
        errorRate: (this.metrics.errorCount / this.metrics.requestCount * 100).toFixed(2),
      },
      database: {
        connections: this.metrics.dbConnections,
      },
    };
  }
}

// Usage
const aggregator = new HealthMetricsAggregator();

app.get('/admin/metrics/health', (req, res) => {
  res.json(aggregator.collectMetrics());
});

Best Practices

Metrics Collection
- Collect right metrics
- Use appropriate intervals
- Label data properly
- Monitor cardinality
Alerting
- Alert on anomalies
- Set smart thresholds
- Avoid alert fatigue
- Document runbooks
Visualization
- Clear dashboards
- Real-time updates
- Historical trends
- Context information
Retention
- Appropriate retention periods
- Archive old data
- Optimize storage
- Maintain access
Integration
- Integrate with incident management
- Link logs to metrics
- Correlate traces
- Cross-service visibility

Real-World Applications

Production Monitoring: Track system health
Performance Analysis: Identify bottlenecks
Capacity Planning: Forecast resource needs
Incident Response: Quick diagnostics
SLA Verification: Prove service levels