Node.js Streams: Processing Large Files Without Running Out of Memory

The Memory Problem // This will OOM on a 2GB file const data = await fs.readFile('huge-file.csv'); // reads entire file into memory const lines = data.toString().split('\n'); // crashes with: JavaScript heap out of memory Streams process data in chunks. You never load the full file—you process pieces as they arrive. Reading Files With Streams import { createReadStream } from 'fs'; import { createInterface } from 'readline'; async function processCSV(filePath: string) { const fileStream = createReadStream(filePath); const rl = createInterface({ input: fileStream, crlfDelay: Infinity }); let lineCount = 0; for await (const line of rl) { // Process one line at a time — never more than ~1KB in memory const [name, email, amount] = line.split(','); await processRecord({ name, email, amount }); lineCount++; } return lineCount; } // A 10GB file uses < 50MB of memory await processCSV('./million-rows.csv'); Transform Streams import { Transform, pipeline } from 'stream'; import { promisify } f