Health Checks and Readiness Probes

Health checks monitor application status. Learn to implement comprehensive health checks and readiness probes for production reliability.

Challenge

Implement sophisticated health checks that provide accurate application status for orchestrators, load balancers, and monitoring systems.

Explanation

Health Check Types

Liveness Check: Is app running?
Readiness Check: Is app ready for traffic?
Startup Check: Has app initialized?
Dependency Check: Are external services available?
Deep Health: Full application validation

Key Aspects

Response Time: Should be fast
Dependencies: Check databases, caches
Resource Usage: Memory, disk, connections
Version: Return app version
Status: Detailed health information

Code Examples

1. Basic Health Checks

const express = require('express');
const app = express();

// Simple liveness probe
app.get('/health', (req, res) => {
  res.json({
    status: 'UP',
    timestamp: new Date().toISOString(),
  });
});

// Readiness probe
app.get('/ready', async (req, res) => {
  try {
    // Check database
    await database.query('SELECT 1');

    // Check cache
    await cache.ping();

    res.json({
      status: 'READY',
      timestamp: new Date().toISOString(),
    });
  } catch (error) {
    res.status(503).json({
      status: 'NOT_READY',
      error: error.message,
    });
  }
});

app.listen(3000);

2. Comprehensive Health Check

class HealthCheckService {
  constructor() {
    this.startTime = Date.now();
    this.checks = new Map();
    this.checks.set('database', this.checkDatabase.bind(this));
    this.checks.set('redis', this.checkRedis.bind(this));
    this.checks.set('disk-space', this.checkDiskSpace.bind(this));
    this.checks.set('memory', this.checkMemory.bind(this));
  }

  async getHealth(detailed = false) {
    const checks = {};
    const results = [];

    for (const [name, check] of this.checks) {
      try {
        const result = await this.executeCheck(check);
        checks[name] = 'UP';
        results.push({ component: name, status: 'UP' });
      } catch (error) {
        checks[name] = 'DOWN';
        results.push({ component: name, status: 'DOWN', error: error.message });
      }
    }

    const overallStatus = Object.values(checks).every(s => s === 'UP') ? 'UP' : 'DOWN';

    return {
      status: overallStatus,
      timestamp: new Date().toISOString(),
      uptime: Math.floor((Date.now() - this.startTime) / 1000),
      version: process.env.APP_VERSION || '1.0.0',
      ...(detailed && {
        checks: results,
        environment: {
          nodeVersion: process.version,
          environment: process.env.NODE_ENV,
        },
      }),
    };
  }

  async checkDatabase() {
    const client = await pool.connect();
    try {
      await client.query('SELECT 1');
    } finally {
      client.release();
    }
  }

  async checkRedis() {
    return cache.ping();
  }

  async checkDiskSpace() {
    const diskUsage = await getDiskUsage();
    if (diskUsage > 90) {
      throw new Error('Low disk space');
    }
  }

  async checkMemory() {
    const memUsage = process.memoryUsage().heapUsed / process.memoryUsage().heapTotal;
    if (memUsage > 0.9) {
      throw new Error('High memory usage');
    }
  }

  async executeCheck(check) {
    return Promise.race([
      check(),
      new Promise((_, reject) =>
        setTimeout(() => reject(new Error('Timeout')), 5000)
      ),
    ]);
  }
}

// Express setup
const healthService = new HealthCheckService();

app.get('/health', async (req, res) => {
  const health = await healthService.getHealth();
  const statusCode = health.status === 'UP' ? 200 : 503;
  res.status(statusCode).json(health);
});

app.get('/health/detailed', async (req, res) => {
  const health = await healthService.getHealth(true);
  const statusCode = health.status === 'UP' ? 200 : 503;
  res.status(statusCode).json(health);
});

3. Kubernetes Health Probes

// Health endpoint configuration
const healthEndpoints = {
  // Startup probe - checks if app has started
  startup: async (req, res) => {
    if (!initialized) {
      return res.status(503).json({ error: 'Still initializing' });
    }
    res.json({ status: 'started' });
  },

  // Readiness probe - checks if app is ready for traffic
  readiness: async (req, res) => {
    try {
      await Promise.all([
        database.query('SELECT 1'),
        cache.ping(),
      ]);
      res.json({ status: 'ready' });
    } catch (error) {
      res.status(503).json({ error: 'Not ready' });
    }
  },

  // Liveness probe - checks if app is alive
  liveness: async (req, res) => {
    // Simple check - just verify app is responding
    if (process.uptime() > 0) {
      res.json({ status: 'alive' });
    } else {
      res.status(503).json({ error: 'Not alive' });
    }
  },
};

app.get('/health/startup', healthEndpoints.startup);
app.get('/health/ready', healthEndpoints.readiness);
app.get('/health/live', healthEndpoints.liveness);

4. Dependency Health Monitoring

class DependencyHealthMonitor {
  constructor() {
    this.dependencies = [];
    this.lastCheckTime = {};
    this.checkInterval = 10000; // 10 seconds
  }

  registerDependency(name, checkFn) {
    this.dependencies.push({ name, checkFn });
  }

  async checkAll() {
    const results = {};

    for (const dep of this.dependencies) {
      const startTime = Date.now();

      try {
        await Promise.race([
          dep.checkFn(),
          this.timeout(5000),
        ]);

        results[dep.name] = {
          status: 'UP',
          responseTime: Date.now() - startTime,
          lastCheck: new Date(),
        };
      } catch (error) {
        results[dep.name] = {
          status: 'DOWN',
          error: error.message,
          lastCheck: new Date(),
        };
      }
    }

    return results;
  }

  timeout(ms) {
    return new Promise((_, reject) =>
      setTimeout(() => reject(new Error('Health check timeout')), ms)
    );
  }

  startContinuousMonitoring() {
    setInterval(async () => {
      const results = await this.checkAll();
      this.updateMetrics(results);
    }, this.checkInterval);
  }

  updateMetrics(results) {
    // Send to monitoring system
    Object.entries(results).forEach(([dep, status]) => {
      const upStatus = status.status === 'UP' ? 1 : 0;
      console.log(`dependency_status{name="${dep}"} ${upStatus}`);
    });
  }
}

// Setup monitoring
const monitor = new DependencyHealthMonitor();

monitor.registerDependency('database', async () => {
  const client = await pool.connect();
  try {
    await client.query('SELECT 1');
  } finally {
    client.release();
  }
});

monitor.registerDependency('redis', async () => {
  await cache.ping();
});

monitor.registerDependency('elasticsearch', async () => {
  await fetch('http://elasticsearch:9200/_cluster/health');
});

monitor.startContinuousMonitoring();

// Endpoint to get current status
app.get('/admin/dependencies', async (req, res) => {
  const status = await monitor.checkAll();
  res.json(status);
});

5. Graceful Shutdown with Health Checks

class HealthAwareShutdown {
  constructor(server) {
    this.server = server;
    this.isHealthy = true;
    this.activeConnections = new Set();
  }

  start() {
    this.server.on('connection', (conn) => {
      this.activeConnections.add(conn);
      conn.on('close', () => this.activeConnections.delete(conn));
    });

    process.on('SIGTERM', () => this.shutdown());
    process.on('SIGINT', () => this.shutdown());
  }

  markUnhealthy() {
    this.isHealthy = false;
    console.log('Marking service as unhealthy');

    // Stop accepting new requests
    this.server.close();
  }

  async shutdown() {
    console.log('Starting graceful shutdown');

    // Mark as unhealthy immediately
    this.markUnhealthy();

    // Wait for active connections
    const shutdownTimeout = new Promise((resolve) => {
      setTimeout(resolve, 30000); // 30 second timeout
    });

    const connectionClose = new Promise((resolve) => {
      const checkInterval = setInterval(() => {
        if (this.activeConnections.size === 0) {
          clearInterval(checkInterval);
          resolve();
        }
      }, 1000);
    });

    await Promise.race([connectionClose, shutdownTimeout]);

    // Force close remaining connections
    this.activeConnections.forEach(conn => conn.destroy());

    process.exit(0);
  }
}

// Setup health checks
app.get('/health', (req, res) => {
  if (!shutdown.isHealthy) {
    return res.status(503).json({ status: 'SHUTTING_DOWN' });
  }
  res.json({ status: 'UP' });
});

const shutdown = new HealthAwareShutdown(server);
shutdown.start();

Best Practices

Check Design
- Keep checks fast (<1 second)
- Use appropriate timeouts
- Check all dependencies
- Provide detailed info
Failure Handling
- Distinguish between types of failures
- Don’t mark unhealthy too quickly
- Implement recovery logic
- Monitor check performance
Integration
- Match orchestrator requirements
- Configure proper thresholds
- Set appropriate intervals
- Log check results
Performance
- Cache check results
- Use connection pooling
- Avoid heavy operations
- Monitor impact
Monitoring
- Track check results
- Alert on failures
- Monitor response times
- Analyze patterns

Real-World Applications

Kubernetes: Liveness and readiness probes
Load Balancers: Route to healthy instances
Monitoring: Track service health
Orchestration: Auto-restart unhealthy services
Graceful Shutdown: Drain traffic before restart