headscale/monitoring-config.yaml

# OIDC Role Mapping Monitoring Configuration
# This file provides monitoring, alerting, and metrics configuration for production deployments

# Prometheus Metrics
prometheus_metrics:
  # Headscale metrics to track
  headscale_metrics:
    - name: "headscale_oidc_logins_total"
      description: "Total number of OIDC logins"
      labels: ["result", "provider"]
    
    - name: "headscale_oidc_groups_extracted_total"
      description: "Number of groups extracted from OIDC claims"
      labels: ["user", "group_count"]
    
    - name: "headscale_users_with_groups_total"
      description: "Total users with group assignments"
      labels: ["group_name"]
  
  # Headplane metrics to track
  headplane_metrics:
    - name: "headplane_role_assignments_total"
      description: "Total role assignments by type"
      labels: ["role", "source"]
    
    - name: "headplane_oidc_role_mappings_total"
      description: "OIDC group to role mappings"
      labels: ["group", "role", "result"]
    
    - name: "headplane_capability_checks_total"
      description: "Capability check results"
      labels: ["capability", "result", "role"]

# Grafana Dashboard Configuration
grafana_dashboard:
  title: "OIDC Role Mapping Dashboard"
  
  panels:
    - title: "OIDC Login Success Rate"
      type: "stat"
      query: "rate(headscale_oidc_logins_total{result='success'}[5m])"
      
    - title: "Role Distribution"
      type: "pie"
      query: "sum by (role) (headplane_role_assignments_total)"
      
    - title: "Group Mapping Success Rate"
      type: "graph"
      query: "rate(headplane_oidc_role_mappings_total{result='success'}[5m])"
      
    - title: "Users by Group"
      type: "table"
      query: "headscale_users_with_groups_total"
      
    - title: "Failed Role Assignments"
      type: "logs"
      query: "headplane_oidc_role_mappings_total{result='failed'}"

# Alerting Rules
alerting_rules:
  groups:
    - name: "oidc_role_mapping"
      rules:
        - alert: "OIDCLoginFailureSpike"
          expr: "rate(headscale_oidc_logins_total{result='failed'}[5m]) > 0.1"
          for: "2m"
          labels:
            severity: "warning"
          annotations:
            summary: "High OIDC login failure rate"
            description: "OIDC login failures have exceeded 10% over the last 5 minutes"
        
        - alert: "RoleMappingFailures"
          expr: "rate(headplane_oidc_role_mappings_total{result='failed'}[5m]) > 0"
          for: "1m"
          labels:
            severity: "critical"
          annotations:
            summary: "Role mapping failures detected"
            description: "Users are failing to get assigned proper roles from OIDC groups"
        
        - alert: "UnexpectedOwnerRoleAssignments"
          expr: "increase(headplane_role_assignments_total{role='owner'}[1h]) > 1"
          for: "0m"
          labels:
            severity: "critical"
          annotations:
            summary: "Multiple owner role assignments detected"
            description: "More than one owner role has been assigned in the last hour"
        
        - alert: "NoGroupsInOIDCTokens"
          expr: "rate(headscale_oidc_groups_extracted_total{group_count='0'}[10m]) > 0.5"
          for: "5m"
          labels:
            severity: "warning"
          annotations:
            summary: "OIDC tokens missing group claims"
            description: "Over 50% of OIDC logins are not receiving group information"

# Log Monitoring Configuration
log_monitoring:
  # Headscale log patterns to monitor
  headscale_patterns:
    - pattern: "Failed to unmarshal user groups"
      severity: "error"
      action: "alert"
      
    - pattern: "Groups extracted from OIDC"
      severity: "info"
      action: "metric"
      
    - pattern: "OIDC callback.*groups.*empty"
      severity: "warning"
      action: "alert"
  
  # Headplane log patterns to monitor
  headplane_patterns:
    - pattern: "Role mapping.*failed"
      severity: "error"
      action: "alert"
      
    - pattern: "User.*assigned role.*owner"
      severity: "info"
      action: "audit_log"
      
    - pattern: "Groups.*not found in mapping"
      severity: "warning"
      action: "metric"

# Security Monitoring
security_monitoring:
  # Track privilege escalation attempts
  privilege_escalation:
    - monitor: "Role changes from member to admin+"
      threshold: 1
      window: "1h"
      
    - monitor: "Direct database role modifications"
      threshold: 0
      window: "5m"
  
  # Monitor suspicious group assignments
  suspicious_groups:
    - pattern: "groups containing 'admin' assigned to new users"
      threshold: 5
      window: "1h"
      
    - pattern: "owner role assigned outside business hours"
      threshold: 0
      window: "24h"

# Health Checks
health_checks:
  oidc_provider:
    - name: "OIDC Issuer Reachability"
      url: "${OIDC_ISSUER}/.well-known/openid-configuration"
      interval: "30s"
      timeout: "5s"
      
    - name: "OIDC Token Endpoint"
      url: "${OIDC_ISSUER}/protocol/openid-connect/token"
      interval: "60s"
      timeout: "10s"
  
  headscale:
    - name: "Headscale Health"
      url: "${HEADSCALE_URL}/health"
      interval: "10s"
      timeout: "3s"
      
    - name: "Headscale OIDC Endpoint"
      url: "${HEADSCALE_URL}/oidc/callback"
      interval: "60s"
      timeout: "5s"
  
  headplane:
    - name: "Headplane UI"
      url: "${HEADPLANE_URL}/admin"
      interval: "30s"
      timeout: "5s"
      
    - name: "Headplane OIDC Callback"
      url: "${HEADPLANE_URL}/admin/oidc/callback"
      interval: "60s"
      timeout: "5s"

# Audit Trail Configuration
audit_trail:
  # Events to log for compliance
  critical_events:
    - "Owner role assignments"
    - "Role mapping configuration changes"
    - "OIDC provider configuration updates"
    - "Database schema migrations"
    - "Manual role overrides"
  
  # Log format for audit events
  log_format:
    timestamp: "ISO8601"
    user_id: "OIDC subject"
    action: "Role assignment/change"
    old_value: "Previous role"
    new_value: "New role"
    source: "OIDC/Manual"
    ip_address: "Client IP"
    user_agent: "Browser/Client info"

# Performance Monitoring
performance_monitoring:
  # Database performance
  database_metrics:
    - "Query execution time for user lookups"
    - "Groups JSON parsing performance"
    - "Database connection pool usage"
    - "Migration execution time"
  
  # OIDC performance
  oidc_metrics:
    - "Token exchange response time"
    - "UserInfo endpoint response time"
    - "Groups claim extraction time"
    - "Role mapping calculation time"

# Incident Response Runbook
incident_response:
  scenarios:
    - name: "OIDC Provider Outage"
      steps:
        1. "Check OIDC provider status page"
        2. "Verify network connectivity"
        3. "Enable fallback authentication if available"
        4. "Communicate to users about temporary limitations"
        5. "Monitor provider restoration"
    
    - name: "Mass Role Assignment Failures"
      steps:
        1. "Check OIDC provider group claim configuration"
        2. "Verify role mapping configuration"
        3. "Check for recent configuration changes"
        4. "Test with known working user account"
        5. "Consider temporary manual role assignments"
    
    - name: "Unexpected Owner Role Assignments"
      steps:
        1. "Immediately review audit logs"
        2. "Verify OIDC group membership in provider"
        3. "Check for compromised accounts"
        4. "Review role mapping configuration"
        5. "Consider temporary role restrictions"

# Backup and Recovery Monitoring
backup_monitoring:
  # Monitor backup processes
  backup_checks:
    - name: "Headscale Database Backup"
      frequency: "daily"
      retention: "30 days"
      verification: "restore test monthly"
      
    - name: "Headplane Database Backup"
      frequency: "daily"
      retention: "30 days"
      verification: "restore test monthly"
      
    - name: "Configuration Backup"
      frequency: "on change"
      retention: "90 days"
      verification: "syntax check"

# Compliance Reporting
compliance_reporting:
  # Generate reports for auditors
  reports:
    - name: "Monthly Role Assignment Report"
      frequency: "monthly"
      includes:
        - "New role assignments"
        - "Role changes"
        - "Group membership changes"
        - "Failed authentication attempts"
    
    - name: "Quarterly Access Review"
      frequency: "quarterly"
      includes:
        - "All users and their assigned roles"
        - "Group to role mapping effectiveness"
        - "Orphaned accounts"
        - "Privilege escalation events"