File Operations

Challenge

Your application needs to read configuration from multiple files, process large files without loading everything in memory, and handle file operations concurrently without blocking. How do you do this efficiently?

Explanation

Node.js fs module provides both synchronous and asynchronous methods for file operations. Understanding the differences and using streams for large files is crucial for performance.

Synchronous vs Asynchronous

const fs = require('fs');

// ❌ Synchronous - blocks event loop
const data = fs.readFileSync('large-file.txt', 'utf8');
console.log('File read'); // Waits for file operation
console.log(data.length);

// ✅ Asynchronous with callback
fs.readFile('large-file.txt', 'utf8', (err, data) => {
  if (err) throw err;
  console.log('File read');
  console.log(data.length);
});

console.log('Reading file...'); // Executes immediately

// ✅ Async/await with Promises
async function readFileAsync() {
  const data = await fs.promises.readFile('large-file.txt', 'utf8');
  console.log('File read');
  return data;
}

readFileAsync().catch(err => console.error(err));

Solution: Multiple Files with Configuration

async function loadConfig() {
  const fs = require('fs').promises;
  const path = require('path');

  // Load configs from multiple files concurrently
  const [appConfig, dbConfig, apiConfig] = await Promise.all([
    fs.readFile(path.join(__dirname, 'config/app.json'), 'utf8')
      .then(data => JSON.parse(data)),
    fs.readFile(path.join(__dirname, 'config/database.json'), 'utf8')
      .then(data => JSON.parse(data)),
    fs.readFile(path.join(__dirname, 'config/api.json'), 'utf8')
      .then(data => JSON.parse(data))
  ]);

  return {
    app: appConfig,
    database: dbConfig,
    api: apiConfig
  };
}

// Usage
loadConfig()
  .then(config => console.log(config))
  .catch(err => console.error('Config load failed:', err));

Working with Streams

const fs = require('fs');

// Stream large files without loading in memory
const readStream = fs.createReadStream('large-file.txt', {
  encoding: 'utf8',
  highWaterMark: 64 * 1024 // 64KB chunks
});

let lineCount = 0;

readStream.on('data', (chunk) => {
  lineCount += chunk.split('\n').length;
});

readStream.on('error', (err) => {
  console.error('Read error:', err);
});

readStream.on('end', () => {
  console.log(`Total lines: ${lineCount}`);
});

// Stream processing with transform
const { Transform } = require('stream');

const uppercase = new Transform({
  transform(chunk, encoding, callback) {
    this.push(chunk.toString().toUpperCase());
    callback();
  }
});

fs.createReadStream('input.txt')
  .pipe(uppercase)
  .pipe(fs.createWriteStream('output.txt'));

File and Directory Operations

const fs = require('fs').promises;
const path = require('path');

// Check if file exists
async function fileExists(filePath) {
  try {
    await fs.access(filePath);
    return true;
  } catch {
    return false;
  }
}

// Create directories
async function ensureDir(dirPath) {
  try {
    await fs.mkdir(dirPath, { recursive: true });
  } catch (err) {
    if (err.code !== 'EEXIST') throw err;
  }
}

// List files in directory
async function listFiles(dirPath) {
  const files = await fs.readdir(dirPath);
  return files;
}

// Get file info
async function getFileInfo(filePath) {
  const stat = await fs.stat(filePath);
  return {
    size: stat.size,
    isFile: stat.isFile(),
    isDirectory: stat.isDirectory(),
    modified: stat.mtime
  };
}

// Copy file
async function copyFile(source, destination) {
  await fs.copyFile(source, destination);
}

// Delete file
async function deleteFile(filePath) {
  await fs.unlink(filePath);
}

// Recursive directory copy
async function copyDir(source, destination) {
  await fs.mkdir(destination, { recursive: true });

  const entries = await fs.readdir(source, { withFileTypes: true });

  for (const entry of entries) {
    const srcPath = path.join(source, entry.name);
    const destPath = path.join(destination, entry.name);

    if (entry.isDirectory()) {
      await copyDir(srcPath, destPath);
    } else {
      await fs.copyFile(srcPath, destPath);
    }
  }
}

Practical Example: Log File Analysis

const fs = require('fs');
const readline = require('readline');

// Process log file line by line
async function analyzeLogs(logPath) {
  const readStream = fs.createReadStream(logPath);

  const rl = readline.createInterface({
    input: readStream,
    crlfDelay: Infinity
  });

  const stats = {
    totalLines: 0,
    errors: 0,
    warnings: 0,
    info: 0
  };

  for await (const line of rl) {
    stats.totalLines++;

    if (line.includes('ERROR')) {
      stats.errors++;
    } else if (line.includes('WARN')) {
      stats.warnings++;
    } else if (line.includes('INFO')) {
      stats.info++;
    }
  }

  return stats;
}

// Usage
analyzeLogs('./app.log')
  .then(stats => console.log(stats))
  .catch(err => console.error(err));

Performance Considerations

// ⚠️ Avoid reading entire large files
const fs = require('fs');

// Bad: 500MB file loaded entirely in memory
fs.readFile('huge-file.bin', (err, data) => {
  // data is entire 500MB in memory
});

// Good: Process with streams
fs.createReadStream('huge-file.bin')
  .on('data', (chunk) => {
    // Process chunk at a time
    processChunk(chunk);
  });

// Good: Use pipes for efficiency
fs.createReadStream('input.txt')
  .pipe(processTransform)
  .pipe(fs.createWriteStream('output.txt'));

Watching Files

const fs = require('fs');

// Watch for file changes
fs.watch('./config.json', (eventType, filename) => {
  if (eventType === 'change') {
    console.log('Config file changed, reloading...');
    // Reload configuration
  }
});

// Recursive watch (with polling)
fs.watch('./src', { recursive: true }, (eventType, filename) => {
  if (filename && filename.endsWith('.js')) {
    console.log(`JavaScript file changed: ${filename}`);
  }
});