Webhooks and Event-Driven APIs

Webhooks enable real-time event delivery to external systems. Learn to design, implement, and manage webhook systems for event-driven architectures.

Challenge Statement

Build a robust webhook system that reliably delivers events to subscribers while handling failures, retries, and ensuring data integrity.

Explanation

Webhooks vs Polling

Webhooks: Server pushes events to clients (real-time) Polling: Clients periodically ask for new data (delayed)

Webhook Characteristics

Push-based: Server initiates communication Real-time: Immediate event delivery Scalable: No polling overhead Reliable: Require retry mechanisms

Code Examples

1. Webhook Registration System

const express = require('express');
const database = require('./database');

const app = express();
app.use(express.json());

// Register webhook
app.post('/api/webhooks', async (req, res) => {
  const { url, events, secret } = req.body;

  if (!url || !events || events.length === 0) {
    return res.status(400).json({ error: 'Invalid webhook data' });
  }

  try {
    const webhook = await database.webhooks.create({
      url,
      events,
      secret,
      userId: req.user.id,
      active: true,
      createdAt: new Date(),
      lastTriggeredAt: null,
    });

    res.status(201).json(webhook);
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

// List webhooks
app.get('/api/webhooks', async (req, res) => {
  const webhooks = await database.webhooks.find({ userId: req.user.id });
  res.json(webhooks);
});

// Delete webhook
app.delete('/api/webhooks/:id', async (req, res) => {
  await database.webhooks.delete(req.params.id);
  res.status(204).send();
});

app.listen(3000);

2. Webhook Delivery with Retries

const axios = require('axios');
const crypto = require('crypto');
const database = require('./database');

class WebhookDelivery {
  constructor() {
    this.maxRetries = 5;
    this.initialDelay = 1000; // 1 second
  }

  generateSignature(payload, secret) {
    return crypto
      .createHmac('sha256', secret)
      .update(JSON.stringify(payload))
      .digest('hex');
  }

  async deliver(webhook, event) {
    let attempt = 0;
    let delay = this.initialDelay;

    while (attempt < this.maxRetries) {
      try {
        const signature = this.generateSignature(event, webhook.secret);

        const response = await axios.post(webhook.url, event, {
          headers: {
            'X-Webhook-Signature': signature,
            'X-Webhook-ID': webhook.id,
            'Content-Type': 'application/json',
          },
          timeout: 10000,
        });

        // Success
        await database.webhookLogs.create({
          webhookId: webhook.id,
          event: event.type,
          status: 'success',
          statusCode: response.status,
          attempt,
          timestamp: new Date(),
        });

        return true;
      } catch (error) {
        attempt++;

        const statusCode = error.response?.status;

        // Don't retry on 4xx errors (except 408, 429)
        if (statusCode && statusCode < 500 && statusCode !== 408 && statusCode !== 429) {
          await database.webhookLogs.create({
            webhookId: webhook.id,
            event: event.type,
            status: 'failed',
            statusCode,
            attempt,
            error: error.message,
            timestamp: new Date(),
          });
          return false;
        }

        if (attempt < this.maxRetries) {
          await new Promise(resolve => setTimeout(resolve, delay));
          delay *= 2; // Exponential backoff
        }
      }
    }

    // Final failure
    await database.webhookLogs.create({
      webhookId: webhook.id,
      event: event.type,
      status: 'failed',
      attempt: this.maxRetries,
      error: 'Max retries exceeded',
      timestamp: new Date(),
    });

    return false;
  }

  async deliverToSubscribers(eventType, data) {
    const webhooks = await database.webhooks.find({ events: eventType, active: true });

    const results = await Promise.allSettled(
      webhooks.map(webhook => this.deliver(webhook, {
        type: eventType,
        data,
        timestamp: new Date(),
      }))
    );

    return results;
  }
}

module.exports = new WebhookDelivery();

3. Event Emitter Pattern

const EventEmitter = require('events');
const webhookDelivery = require('./webhookDelivery');

class EventDispatcher extends EventEmitter {
  constructor() {
    super();
    this.setupListeners();
  }

  setupListeners() {
    this.on('user.created', async (data) => {
      await webhookDelivery.deliverToSubscribers('user.created', data);
    });

    this.on('user.updated', async (data) => {
      await webhookDelivery.deliverToSubscribers('user.updated', data);
    });

    this.on('user.deleted', async (data) => {
      await webhookDelivery.deliverToSubscribers('user.deleted', data);
    });

    this.on('order.created', async (data) => {
      await webhookDelivery.deliverToSubscribers('order.created', data);
    });
  }

  async emitAsync(eventType, data) {
    return new Promise((resolve, reject) => {
      this.emit(eventType, data);
      // Resolve immediately, delivery happens asynchronously
      resolve();
    });
  }
}

// Usage in application
const eventDispatcher = new EventDispatcher();

app.post('/api/users', async (req, res) => {
  const user = await database.users.create(req.body);

  // Emit event (non-blocking)
  eventDispatcher.emitAsync('user.created', {
    userId: user.id,
    email: user.email,
    createdAt: user.createdAt,
  }).catch(err => logger.error('Event emission failed', err));

  res.status(201).json(user);
});

module.exports = eventDispatcher;

4. Webhook Verification

const crypto = require('crypto');

function verifyWebhookSignature(req, secret) {
  const signature = req.headers['x-webhook-signature'];
  const body = req.rawBody; // Need to capture raw body

  if (!signature) {
    return false;
  }

  const expectedSignature = crypto
    .createHmac('sha256', secret)
    .update(body)
    .digest('hex');

  return crypto.timingSafeEqual(
    Buffer.from(signature),
    Buffer.from(expectedSignature)
  );
}

// Middleware to verify webhooks
function verifyWebhookMiddleware(req, res, next) {
  // Capture raw body
  let rawBody = '';
  req.setEncoding('utf8');

  req.on('data', chunk => {
    rawBody += chunk;
  });

  req.on('end', () => {
    req.rawBody = rawBody;
    req.body = JSON.parse(rawBody);
    next();
  });
}

// Webhook endpoint
app.post('/webhooks/inbound', verifyWebhookMiddleware, (req, res) => {
  const secret = process.env.WEBHOOK_SECRET;

  if (!verifyWebhookSignature(req, secret)) {
    return res.status(401).json({ error: 'Invalid signature' });
  }

  // Process webhook
  console.log('Webhook received:', req.body);
  res.json({ received: true });
});

5. Webhook Testing Framework

class WebhookTester {
  constructor() {
    this.receivedEvents = [];
    this.server = null;
  }

  async start(port = 9999) {
    const express = require('express');
    const app = express();
    app.use(express.json());

    app.post('/webhook', (req, res) => {
      this.receivedEvents.push({
        event: req.body,
        receivedAt: new Date(),
        headers: req.headers,
      });
      res.json({ received: true });
    });

    return new Promise(resolve => {
      this.server = app.listen(port, () => {
        console.log(`Webhook test server running on port ${port}`);
        resolve();
      });
    });
  }

  getEvents(eventType = null) {
    if (!eventType) {
      return this.receivedEvents;
    }
    return this.receivedEvents.filter(e => e.event.type === eventType);
  }

  async stop() {
    return new Promise(resolve => {
      this.server.close(resolve);
    });
  }

  reset() {
    this.receivedEvents = [];
  }
}

module.exports = WebhookTester;

Best Practices

Security
- Sign webhooks with HMAC
- Verify signatures on receipt
- Use HTTPS only
- Validate event sources
Reliability
- Implement exponential backoff retries
- Set appropriate timeouts
- Log all delivery attempts
- Monitor failed deliveries
Performance
- Use message queues for delivery
- Batch events when appropriate
- Implement rate limiting per subscriber
- Decouple delivery from main request
Developer Experience
- Provide webhook testing tools
- Clear documentation with examples
- Event replay capabilities
- Dashboard for monitoring
Scalability
- Use background job queues
- Implement webhook throttling
- Scale delivery workers independently
- Archive old webhook logs

Real-World Applications

Payment Processing: Payment status updates
E-Commerce: Order and inventory updates
SaaS Integration: User and account events
Notifications: Real-time alerts and notifications
Analytics: Event tracking and data collection