Hot Partition in Distributed Databases — When One Shard Gets All the Heat

Bad partition key: user_country = "US"

Shard 1 (US):   ████████████████████  90% of traffic
Shard 2 (EU):   ████  7%
Shard 3 (APAC): █    3%

Most data + most traffic → shard 1 is HOT

// DynamoDB example: celebrity with 10M followers writing to same partition
const SHARD_COUNT = 20

// ❌ Hot partition: all reads/writes for celebrity go to same partition
const hotKey = `user#${celebId}`

// ✅ Spread reads/writes across 20 sub-partitions
function getShardedKey(userId: string): string {
  const shard = Math.floor(Math.random() * SHARD_COUNT)
  return `user#${userId}#${shard}`
}

// Write: random shard
async function writePost(userId: string, post: Post) {
  const key = getShardedKey(userId)
  await dynamodb.put({ TableName: 'posts', Item: { pk: key, ...post } })
}

// Read all: scatter-gather across all shards
async function getAllPosts(userId: string): Promise<Post[]> {
  const keys = Array.from({ length: SHARD_COUNT }, (_, i) =>
    `user#${userId}#${i}`
  )

  const results = await Promise.all(
    keys.map(key =>
      dynamodb.query({ TableName: 'posts', KeyConditionExpression: 'pk = :pk',
        ExpressionAttributeValues: { ':pk': key } })
    )
  )

  return results.flatMap(r => r.Items as Post[])
}

// ❌ Sequential timestamp → all writes go to "latest" partition
const key = `events#${Date.now()}`  // Always the same shard!

// ✅ Append random shard bucket
const WRITE_SHARDS = 16

function getTimeSeriesKey(timestamp: number): string {
  const bucket = timestamp % WRITE_SHARDS  // 0-15
  return `events#${bucket}#${timestamp}`
}

// Cassandra example
async function insertEvent(event: Event) {
  const bucket = event.timestamp % WRITE_SHARDS
  await cassandra.execute(
    'INSERT INTO events (bucket, ts, data) VALUES (?, ?, ?)',
    [bucket, event.timestamp, event.data]
  )
}

// Read: query all buckets in parallel
async function getEvents(startTs: number, endTs: number) {
  const queries = Array.from({ length: WRITE_SHARDS }, (_, i) =>
    cassandra.execute(
      'SELECT * FROM events WHERE bucket = ? AND ts >= ? AND ts <= ?',
      [i, startTs, endTs]
    )
  )
  const results = await Promise.all(queries)
  return results.flatMap(r => r.rows).sort((a, b) => a.ts - b.ts)
}

import LRU from 'lru-cache'

const hotCache = new LRU<string, any>({
  max: 10_000,
  ttl: 5_000,  // 5 second TTL — fresh enough, but protects DB
})

async function getProductListing(productId: string) {
  // Check in-process cache first (no network hop)
  const cached = hotCache.get(productId)
  if (cached) return cached

  // Check Redis
  const redisCached = await redis.get(`product:${productId}`)
  if (redisCached) {
    const data = JSON.parse(redisCached)
    hotCache.set(productId, data)
    return data
  }

  // DB query — this is the hot partition
  const product = await db.products.findById(productId)
  await redis.setex(`product:${productId}`, 30, JSON.stringify(product))
  hotCache.set(productId, product)
  return product
}

// DynamoDB automatically redistributes capacity to hot partitions
// Enable "adaptive capacity" in your table settings

// For application-level help:
const client = new DynamoDBClient({
  retryStrategy: {
    maxAttempts: 5,
    // Exponential backoff handles ProvisionedThroughputExceededException
  }
})

// Monitor for hot partitions
const metrics = await cloudwatch.getMetricStatistics({
  MetricName: 'ThrottledRequests',
  Namespace: 'AWS/DynamoDB',
  // High throttling on one table/partition = hot partition signal
})

// ❌ These keys land on different nodes — no co-location
const key1 = `user:${userId}:profile`
const key2 = `user:${userId}:feed`

// ✅ Use hash tags {} to force same slot
const key1 = `{user:${userId}}:profile`  // Both hash on userId
const key2 = `{user:${userId}}:feed`     // Same node!

// Now you can MULTI/EXEC across these keys — same slot required

# DynamoDB — check consumed capacity per partition via CloudWatch
aws cloudwatch get-metric-statistics \
  --metric-name ConsumedWriteCapacityUnits \
  --namespace AWS/DynamoDB

# Cassandra — check node load imbalance
nodetool status
# If one node has 60%+ of data → hot partition

# Redis Cluster — check slot distribution
redis-cli --cluster info localhost:7000

# Application-level monitoring
const partitionHitCounts = new Map<string, number>()

function trackPartitionHit(key: string) {
  const partition = getPartitionKey(key)
  const count = (partitionHitCounts.get(partition) ?? 0) + 1
  partitionHitCounts.set(partition, count)

  // Alert if one partition is getting > 50% of traffic
  const total = [...partitionHitCounts.values()].reduce((a, b) => a + b, 0)
  if (count / total > 0.5) {
    logger.warn(`Hot partition detected: ${partition} getting ${((count/total)*100).toFixed(1)}% of traffic`)
  }
}