Service Mesh With Istio — mTLS, Traffic Management, and Observability for Free

# What Istio provides (for free, transparently):
# ✓ Automatic mTLS between all services
# ✓ Traffic policies (canary, circuit breakers, retries)
# ✓ Observability (distributed tracing, metrics)
# ✓ Security policies (RBAC, network policies)
# ✓ Load balancing strategies

# What you pay for:
# ✗ Resource overhead: 1 sidecar proxy per pod (~50MB RAM each)
# ✗ Operational complexity: New debugging layer, YAML config
# ✗ Latency: Proxies add ~10-50ms per hop (variable)
# ✗ Learning curve: Understanding networking gets harder
# ✗ Troubleshooting: Network issues now invisible inside proxies

# Istio is worth it when:
# - You have 20+ microservices needing mTLS
# - You require sophisticated traffic policies
# - Your team has Kubernetes expertise
# - You need enterprise features (traffic splitting, fault injection)
# - You're not resource-constrained

# Istio is overkill when:
# - You have < 10 services
# - Simple HTTP load balancing is sufficient
# - You're resource-constrained (serverless, edge)
# - Your team is new to Kubernetes
# - You can use simpler tools (Kubernetes Network Policies + TLS in app)

# 1. Install Istio
# curl -L https://istio.io/downloadIstio | sh
# cd istio-x.y.z
# export PATH=$PWD/bin:$PATH
# istioctl install --set profile=demo -y

# 2. Enable sidecar injection for a namespace
apiVersion: v1
kind: Namespace
metadata:
  name: production
  labels:
    istio-injection: enabled  # Enables automatic sidecar injection
---

# 3. Verify sidecars are injected
# kubectl get pods -n production
# Should show 2/2 ready (app container + istio-proxy)

# 4. Check injected pod
# kubectl get pod <pod-name> -n production -o jsonpath='{.spec.containers[*].name}'
# Output: app-container istio-proxy

# Manual sidecar injection (if automatic disabled):
istioctl kube-inject -f deployment.yaml | kubectl apply -f -

apiVersion: v1
kind: Pod
metadata:
  name: app-pod
  namespace: production
spec:
  containers:
  - name: app
    image: myapp:latest
    ports:
    - containerPort: 8080
    lifecycle:
      preStop:
        exec:
          command: ["/bin/sh", "-c", "sleep 15"]

  # Automatically injected
  - name: istio-proxy
    image: istio/proxyv2:latest
    resources:
      requests:
        memory: "50Mi"
        cpu: "100m"
      limits:
        memory: "100Mi"
        cpu: "200m"
    env:
    - name: ISTIO_META_WORKLOAD_NAME
      value: "app"
    securityContext:
      allowPrivilegeEscalation: false
      readOnlyRootFilesystem: true
      runAsNonRoot: true
      capabilities:
        drop:
        - ALL

status:
  containerStatuses:
  - name: app
    ready: true
  - name: istio-proxy
    ready: true

# 1. Create PeerAuthentication policy for mTLS
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: default
  namespace: production
spec:
  mtls:
    mode: STRICT  # Require mTLS on all connections
---

# 2. Create DestinationRule to enforce MTLS client config
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: services
  namespace: production
spec:
  host: "*.production.svc.cluster.local"
  trafficPolicy:
    tls:
      mode: ISTIO_MUTUAL  # Use Istio's automatic mTLS
    connectionPool:
      tcp:
        maxConnections: 1000
      http:
        http1MaxPendingRequests: 1000
        maxRequestsPerConnection: 2
---

# 3. Create AuthorizationPolicy to control access
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: api-policy
  namespace: production
spec:
  selector:
    matchLabels:
      app: api
  rules:
  # Allow traffic from frontend
  - from:
    - source:
        principals: ["cluster.local/ns/production/sa/frontend"]
    to:
    - operation:
        methods: ["GET", "POST"]
        paths: ["/api/*"]

  # Allow health checks (bypass mTLS)
  - from:
    - source:
        namespaces: ["istio-system"]
    to:
    - operation:
        ports: ["8081"]
---

# 4. Verify mTLS is working
# kubectl exec -it <pod> -n production -c istio-proxy -- \
#   openssl s_client -connect <service>:8080 -showcerts

# Check if mTLS is enforced
kubectl get peerauthentication -n production

# Verify DestinationRules
kubectl get destinationrules -n production

# Check AuthorizationPolicy
kubectl get authorizationpolicies -n production

# View certificate details
kubectl exec <pod> -n production -c istio-proxy -- \
  cat /etc/certs/out/cert-chain.pem | openssl x509 -text -noout

# Monitor mTLS metrics (if Prometheus is running)
# Query: envoy_listener_ssl_socket_factory_downstream_tls_context_update_failure

# 1. Create multiple versions of deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api-v1
  namespace: production
spec:
  selector:
    matchLabels:
      app: api
      version: v1
  template:
    metadata:
      labels:
        app: api
        version: v1
    spec:
      containers:
      - name: api
        image: myapi:1.0
---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: api-v2
  namespace: production
spec:
  selector:
    matchLabels:
      app: api
      version: v2
  template:
    metadata:
      labels:
        app: api
        version: v2
    spec:
      containers:
      - name: api
        image: myapi:2.0
---

# 2. Create Service (single service for both versions)
apiVersion: v1
kind: Service
metadata:
  name: api
  namespace: production
spec:
  selector:
    app: api
  ports:
  - port: 8080
    targetPort: 8080
---

# 3. Create DestinationRule (subset = version)
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: api
  namespace: production
spec:
  host: api
  trafficPolicy:
    connectionPool:
      http:
        http1MaxPendingRequests: 100
        http2MaxRequests: 1000
  subsets:
  - name: v1
    labels:
      version: v1
  - name: v2
    labels:
      version: v2
---

# 4. Create VirtualService to split traffic
# Canary: 90% to v1, 10% to v2
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: api
  namespace: production
spec:
  hosts:
  - api
  http:
  # Route 10% to v2 (canary)
  - match:
    - uri:
        prefix: "/"
    route:
    - destination:
        host: api
        subset: v1
      weight: 90
    - destination:
        host: api
        subset: v2
      weight: 10
    timeout: 10s
    retries:
      attempts: 3
      perTryTimeout: 5s
---

# 5. Gradually increase v2 traffic
# 0-5 min:  v2 = 10%
# 5-10 min: v2 = 25%
# 10-15 min: v2 = 50%
# 15-20 min: v2 = 100%

# Update VirtualService weights gradually
kubectl patch virtualservice api -n production --type merge \
  -p '{"spec":{"http":[{"route":[{"destination":{"host":"api","subset":"v1"},"weight":75},{"destination":{"host":"api","subset":"v2"},"weight":25}]}]}}'

apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: api-circuit-breaker
  namespace: production
spec:
  host: api
  trafficPolicy:
    # Connection pool limits
    connectionPool:
      tcp:
        maxConnections: 100  # Max concurrent connections
      http:
        http1MaxPendingRequests: 100  # Max pending requests
        http2MaxRequests: 1000  # Max concurrent HTTP/2 streams
        maxRequestsPerConnection: 2  # Keep connections fresh

    # Outlier detection (circuit breaker)
    outlierDetection:
      consecutiveErrors: 5  # Eject after 5 consecutive errors
      interval: 30s  # Check every 30 seconds
      baseEjectionTime: 30s  # Eject for 30 seconds
      maxEjectionPercent: 50  # Eject max 50% of instances
      minRequestVolume: 10  # Only eject if >10 requests in interval

      # HTTP-specific outlier detection
      consecutiveGatewayErrors: 5
      splitExternalLocalOriginErrors: true  # Treat local/external errors separately

      # TCP-specific
      consecutiveConnectFailure: 5

  subsets:
  - name: default
    labels:
      app: api
---

# Monitor circuit breaker status
# Check Envoy stats: envoy_cluster_circuit_breakers_*
# kubectl exec <pod> -c istio-proxy -- curl localhost:15000/stats | grep circuit_breakers

# View which endpoints are ejected
kubectl exec <pod> -c istio-proxy -- curl localhost:15000/clusters | grep outlier

# Expected output when endpoint ejected:
# ::default_priority::100.0.0.1:8080::cx_active::1
# ::default_priority::100.0.0.1:8080::rq_pending::0
# ::default_priority::100.0.0.1:8080::rq_active::0
# ::default_priority::100.0.0.1:8080::healthy::false  <- Ejected!

# Prometheus query for circuit breaker metrics
# envoy_cluster_outlier_detection_ejections_enforced_consecutive_5xx

# 1. Install Jaeger for tracing
kubectl apply -f https://raw.githubusercontent.com/jaegertracing/jaeger-kubernetes/main/jaeger-production-template.yml

# 2. Configure Istio to send traces to Jaeger
apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: tracing-config
  namespace: istio-system
spec:
  tracing:
  - providers:
    - name: jaeger
    randomSamplingPercentage: 10  # Sample 10% of traces
    useRequestIdForTraceSampling: true  # Use consistent sampling

---

# 3. Deploy Jaeger agent
apiVersion: v1
kind: Service
metadata:
  name: jaeger
  namespace: istio-system
spec:
  selector:
    app: jaeger
  ports:
  - port: 6831
    protocol: UDP
    targetPort: 6831

---

# 4. Verify trace headers are propagated
# Check request headers in Jaeger dashboard
# traceparent, b3, x-cloud-trace-context, jaeger-traceid, etc.

// Application code doesn't need to do anything
// Traces are auto-injected by Istio proxies

// But you can still create custom spans
import { trace } from '@opentelemetry/api';

const tracer = trace.getTracer('my-service');
const span = tracer.startSpan('custom-operation');

try {
  // Perform operation
  span.setStatus({ code: SpanStatusCode.OK });
} catch (error) {
  span.setStatus({ code: SpanStatusCode.ERROR });
  span.recordException(error);
} finally {
  span.end();
}

// Trace will show in Jaeger with automatic service-to-service spans

# Scenarios where a service mesh adds complexity without value:

# 1. Few services (< 10)
# - Complexity not justified
# - Use Kubernetes Network Policies + app-level TLS

# 2. Resource constrained
# - Each proxy takes ~50MB RAM
# - 50 pods = 2.5GB just for proxies
# - Use sidecarless proxies (Ambient mode) or skip entirely

# 3. Team learning Kubernetes
# - Service mesh adds another networking layer
# - Start with basic Kubernetes networking
# - Add mesh after team expertise grows

# 4. Simple applications
# - No need for sophisticated traffic policies
# - Load balancer + TLS sufficient
# - Don't add complexity for hypothetical future needs

# 5. Performance sensitive
# - Proxy latency: 10-50ms per hop
# - Can be deal-breaker for latency-critical services
# - Measure before committing

# Best practice: Start simple
# 1. Use Kubernetes DNS and Services
# 2. Add Kubernetes Network Policies for security
# 3. Use app-level TLS/mutual auth if needed
# 4. Only add service mesh when managing 20+ services with complex patterns