AI Audit Logging — Compliance, Debugging, and Accountability for LLM Systems

Introduction

A user claims your AI system violated their privacy. You have no audit trail. You can''t prove what happened. Legal disputes you.

Or your LLM misbehaves in production. You''re left with sparse logs that don''t tell you which prompt or model caused the issue. Debugging becomes guesswork.

Audit logging solves both problems: accountability for regulators, debugging for engineers.

Structured Logging Schema
Querying Logs for Debugging
Cost Audit Trail
Log Retention and GDPR Deletion
Compliance Report Generation
Conclusion

Structured Logging Schema

Design logs that capture everything needed for compliance and debugging:

interface AIAuditLog {
  timestamp: Date;
  requestId: string;
  userId: string;
  sessionId: string;
  
  // LLM details
  modelName: string;
  prompt: string;
  response: string;
  promptTokens: number;
  responseTokens: number;
  costUSD: number;
  latencyMs: number;
  
  // System details
  endpoint: string;
  feature: string;
  version: string;
  
  // Outcomes
  success: boolean;
  errorMessage?: string;
  responseQuality?: number;
  
  // Audit trail
  dataClassification: "public" | "internal" | "confidential" | "restricted";
  piiDetected: boolean;
  piiTypes?: string[];
  userConsent: boolean;
  
  // Compliance
  complianceFramework?: "gdpr" | "hipaa" | "sox" | "ccpa";
  dataResidency: string; // e.g., "US-EAST-1"
  
  // Security
  injectionAttemptDetected: boolean;
  securityScore: number;
}

class StructuredAuditLogger {
  async logAIRequest(log: AIAuditLog): Promise<void> {
    // Validate required fields
    if (!log.requestId || !log.userId) {
      throw new Error("Missing required audit log fields");
    }

    // Scrub PII from logs before storage
    const sanitized = this.sanitizeForLogging(log);

    // Store in immutable append-only log
    await this.persistLog(sanitized);

    // For sensitive operations, also alert
    if (log.dataClassification === "restricted" || log.piiDetected) {
      await this.alertSecurityTeam(sanitized);
    }
  }

  private sanitizeForLogging(log: AIAuditLog): AIAuditLog {
    // Redact PII from prompt and response before logging
    const sanitized = { ...log };

    if (log.piiDetected) {
      // Replace actual PII with placeholders
      sanitized.prompt = this.redactPII(log.prompt);
      sanitized.response = this.redactPII(log.response);
    }

    return sanitized;
  }

  private redactPII(text: string): string {
    // Replace email, phone, SSN, etc with [REDACTED_*]
    return text
      .replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, "[REDACTED_EMAIL]")
      .replace(/\b\d{3}-\d{2}-\d{4}\b/g, "[REDACTED_SSN]")
      .replace(/\b\d{3}[-.]?\d{3}[-.]?\d{4}\b/g, "[REDACTED_PHONE]");
  }

  private async persistLog(log: AIAuditLog): Promise<void> {
    // Store in DynamoDB, PostgreSQL with PITR, or immutable log service
  }

  private async alertSecurityTeam(log: AIAuditLog): Promise<void> {
    // Send real-time alert for restricted data
  }
}

export { StructuredAuditLogger, AIAuditLog };

Querying Logs for Debugging

Structure logs so you can quickly find and analyze failures:

interface LogQueryResult {
  totalLogs: number;
  averageLatency: number;
  errorRate: number;
  costSummary: {
    totalCost: number;
    averageCostPerRequest: number;
    maxCost: number;
  };
  topErrors: Array<{ error: string; count: number }>;
}

class AuditLogQuery {
  async queryByUser(userId: string, limit: number = 1000): Promise<AIAuditLog[]> {
    // SELECT * FROM audit_logs WHERE user_id = ? ORDER BY timestamp DESC LIMIT ?
    return [];
  }

  async queryBySession(sessionId: string): Promise<AIAuditLog[]> {
    // Get all logs for a single user session for end-to-end debugging
    return [];
  }

  async queryByFeature(
    feature: string,
    startDate: Date,
    endDate: Date
  ): Promise<LogQueryResult> {
    const logs = await this.queryLogs({
      feature,
      startDate,
      endDate,
    });

    const errors = logs.filter((l) => !l.success);
    const latencies = logs.map((l) => l.latencyMs);
    const costs = logs.map((l) => l.costUSD);

    return {
      totalLogs: logs.length,
      averageLatency:
        latencies.reduce((a, b) => a + b, 0) / latencies.length,
      errorRate: errors.length / logs.length,
      costSummary: {
        totalCost: costs.reduce((a, b) => a + b, 0),
        averageCostPerRequest: costs.reduce((a, b) => a + b, 0) / costs.length,
        maxCost: Math.max(...costs),
      },
      topErrors: this.aggregateErrors(errors),
    };
  }

  async queryByErrorType(
    errorPattern: RegExp,
    startDate: Date,
    endDate: Date
  ): Promise<AIAuditLog[]> {
    const logs = await this.queryLogs({ startDate, endDate });
    return logs.filter((l) => errorPattern.test(l.errorMessage || ""));
  }

  async queryByModel(
    modelName: string,
    startDate: Date,
    endDate: Date
  ): Promise<LogQueryResult> {
    const logs = await this.queryLogs({
      modelName,
      startDate,
      endDate,
    });

    return this.analyzeLogSet(logs);
  }

  private async queryLogs(filters: Partial<AIAuditLog>): Promise<AIAuditLog[]> {
    // Query database with filters
    return [];
  }

  private aggregateErrors(logs: AIAuditLog[]): Array<{
    error: string;
    count: number;
  }> {
    const counts = new Map<string, number>();

    logs.forEach((log) => {
      const error = log.errorMessage || "unknown";
      counts.set(error, (counts.get(error) || 0) + 1);
    });

    return Array.from(counts.entries())
      .map(([error, count]) => ({ error, count }))
      .sort((a, b) => b.count - a.count)
      .slice(0, 10);
  }

  private analyzeLogSet(logs: AIAuditLog[]): LogQueryResult {
    return {
      totalLogs: logs.length,
      averageLatency: 0,
      errorRate: 0,
      costSummary: { totalCost: 0, averageCostPerRequest: 0, maxCost: 0 },
      topErrors: [],
    };
  }
}

export { AuditLogQuery, LogQueryResult };

Cost Audit Trail

Track spending with full accountability:

interface CostAuditEntry {
  timestamp: Date;
  requestId: string;
  userId: string;
  feature: string;
  modelName: string;
  inputTokens: number;
  outputTokens: number;
  costUSD: number;
  billableUser: boolean;
}

class CostAuditTrail {
  async generateMonthlyReport(
    year: number,
    month: number
  ): Promise<{
    totalCost: number;
    costByFeature: Record<string, number>;
    costByUser: Array<{ userId: string; cost: number }>;
    costByModel: Record<string, number>;
  }> {
    const entries = await this.getCostEntries(year, month);

    const costByFeature = new Map<string, number>();
    const costByUser = new Map<string, number>();
    const costByModel = new Map<string, number>();
    let totalCost = 0;

    for (const entry of entries) {
      if (!entry.billableUser) continue;

      totalCost += entry.costUSD;

      costByFeature.set(
        entry.feature,
        (costByFeature.get(entry.feature) || 0) + entry.costUSD
      );
      costByUser.set(
        entry.userId,
        (costByUser.get(entry.userId) || 0) + entry.costUSD
      );
      costByModel.set(
        entry.modelName,
        (costByModel.get(entry.modelName) || 0) + entry.costUSD
      );
    }

    return {
      totalCost,
      costByFeature: Object.fromEntries(costByFeature),
      costByUser: Array.from(costByUser.entries())
        .map(([userId, cost]) => ({ userId, cost }))
        .sort((a, b) => b.cost - a.cost),
      costByModel: Object.fromEntries(costByModel),
    };
  }

  private async getCostEntries(year: number, month: number): Promise<CostAuditEntry[]> {
    // Query audit logs for the specified month
    return [];
  }
}

export { CostAuditTrail, CostAuditEntry };

Implement retention policies and right-to-be-forgotten compliance:

interface LogRetentionPolicy {
  dataClassification: string;
  retentionDays: number;
  allowsAnonymization: boolean;
  requiresExplicitDeletion: boolean;
}

class LogRetentionManager {
  private policies: Record<string, LogRetentionPolicy> = {
    public: { dataClassification: "public", retentionDays: 365, allowsAnonymization: true, requiresExplicitDeletion: false },
    internal: { dataClassification: "internal", retentionDays: 90, allowsAnonymization: false, requiresExplicitDeletion: false },
    confidential: { dataClassification: "confidential", retentionDays: 30, allowsAnonymization: false, requiresExplicitDeletion: true },
    restricted: { dataClassification: "restricted", retentionDays: 7, allowsAnonymization: false, requiresExplicitDeletion: true },
  };

  async deleteUserLogs(userId: string): Promise<void> {
    // GDPR right to be forgotten
    const logs = await this.queryLogsByUser(userId);

    for (const log of logs) {
      const policy = this.policies[log.dataClassification];

      if (policy.requiresExplicitDeletion) {
        // Securely delete
        await this.secureDelete(log.requestId);
      } else if (policy.allowsAnonymization) {
        // Anonymize instead of delete
        await this.anonymizeLog(log.requestId);
      }
    }
  }

  async purgeExpiredLogs(): Promise<void> {
    const now = new Date();

    for (const classification of Object.keys(this.policies)) {
      const policy = this.policies[classification];
      const cutoffDate = new Date(
        now.getTime() - policy.retentionDays * 24 * 60 * 60 * 1000
      );

      // DELETE FROM audit_logs WHERE data_classification = ? AND timestamp < ?
      await this.deleteLogsOlderThan(classification, cutoffDate);
    }
  }

  private async queryLogsByUser(userId: string): Promise<AIAuditLog[]> {
    return [];
  }

  private async secureDelete(requestId: string): Promise<void> {
    // Multi-pass overwrite before deleting
  }

  private async anonymizeLog(requestId: string): Promise<void> {
    // Replace identifiers with hashes
  }

  private async deleteLogsOlderThan(classification: string, date: Date): Promise<void> {
    // Database cleanup
  }
}

export { LogRetentionManager, LogRetentionPolicy };

Compliance Report Generation

Generate audit reports for regulators and internal compliance teams:

interface ComplianceReport {
  period: { startDate: Date; endDate: Date };
  framework: "gdpr" | "hipaa" | "sox" | "ccpa";
  dataAccessed: number;
  piiExposures: number;
  unauthorizedAccess: number;
  dataBreaches: number;
  retentionCompliance: boolean;
  consentCompliance: boolean;
  auditTrailIntegrity: boolean;
}

class ComplianceReportGenerator {
  async generateGDPRReport(
    startDate: Date,
    endDate: Date
  ): Promise<ComplianceReport> {
    const logs = await this.queryLogs(startDate, endDate);

    const piiExposures = logs.filter((l) => l.piiDetected && !l.userConsent).length;
    const unauthorizedAccess = logs.filter((l) => !l.success && l.errorMessage?.includes("unauthorized")).length;
    const dataBreaches = await this.checkForDataBreaches(startDate, endDate);

    return {
      period: { startDate, endDate },
      framework: "gdpr",
      dataAccessed: logs.length,
      piiExposures,
      unauthorizedAccess,
      dataBreaches,
      retentionCompliance: await this.checkRetentionCompliance(),
      consentCompliance: logs.every((l) => l.userConsent),
      auditTrailIntegrity: await this.verifyAuditTrailIntegrity(),
    };
  }

  async generateHIPAAReport(
    startDate: Date,
    endDate: Date
  ): Promise<ComplianceReport> {
    // HIPAA is stricter on PHI (Protected Health Information)
    const logs = await this.queryLogs(startDate, endDate);
    const phiLogs = logs.filter((l) => l.dataClassification === "restricted");

    return {
      period: { startDate, endDate },
      framework: "hipaa",
      dataAccessed: phiLogs.length,
      piiExposures: phiLogs.filter((l) => !l.userConsent).length,
      unauthorizedAccess: phiLogs.filter((l) => !l.success).length,
      dataBreaches: 0,
      retentionCompliance: await this.checkRetentionCompliance(),
      consentCompliance: phiLogs.every((l) => l.userConsent),
      auditTrailIntegrity: await this.verifyAuditTrailIntegrity(),
    };
  }

  private async queryLogs(startDate: Date, endDate: Date): Promise<AIAuditLog[]> {
    return [];
  }

  private async checkForDataBreaches(startDate: Date, endDate: Date): Promise<number> {
    // Count suspected breaches based on anomalies
    return 0;
  }

  private async checkRetentionCompliance(): Promise<boolean> {
    // Verify retention policies are enforced
    return true;
  }

  private async verifyAuditTrailIntegrity(): Promise<boolean> {
    // Check for gaps or tampering in audit trail
    return true;
  }
}

export { ComplianceReportGenerator, ComplianceReport };

Conclusion

AI audit logging is non-negotiable for regulated industries and good practice everywhere. Log structured data (prompts, responses, costs, metadata), query logs efficiently for debugging, maintain compliance with retention policies and right-to-be-forgotten, and generate reports for regulators.

Treat audit logs as immutable records. They''re your defense against liability, your debugging tool, and your compliance backbone.