Kafka with Node.js — Event Streaming Guide

npm install kafkajs

import { Kafka } from "kafkajs";

const kafka = new Kafka({
  clientId: "my-app",
  brokers: ["localhost:9092"],
});

const producer = kafka.producer();
await producer.connect();

// Produce message
await producer.send({
  topic: "user-events",
  messages: [
    {
      key: "user-123",
      value: JSON.stringify({ type: "user-created", userId: 123 }),
    },
  ],
});

// Batch messages
await producer.send({
  topic: "user-events",
  messages: [
    { key: "user-1", value: JSON.stringify({ type: "user-created" }) },
    { key: "user-2", value: JSON.stringify({ type: "user-created" }) },
  ],
});

await producer.disconnect();

const consumer = kafka.consumer({ groupId: "my-group" });
await consumer.connect();
await consumer.subscribe({ topic: "user-events" });

await consumer.run({
  eachMessage: async ({ topic, partition, message }) => {
    console.log({
      topic,
      partition,
      key: message.key?.toString(),
      value: message.value?.toString(),
    });

    const event = JSON.parse(message.value!.toString());
    // Process event
  },
});

// Multiple consumers same group = load balanced
// Different groups = all get same messages

const consumer1 = kafka.consumer({ groupId: "analytics" });
const consumer2 = kafka.consumer({ groupId: "notifications" });

// Both receive all messages independently

// Produce with key (determines partition)
await producer.send({
  topic: "transactions",
  messages: [
    { key: "user-123", value: "..." }, // Always same partition
    { key: "user-456", value: "..." }, // Different partition
  ],
});

// Guarantees ordering within partition
// Allows parallel processing across partitions