instruction: Create a monitoring setup with Prometheus, Grafana, and AlertManager for Kubernetes
output: Here's a comprehensive monitoring setup for Kubernetes:
  
  ```yaml
  # prometheus-namespace.yaml
  apiVersion: v1
  kind: Namespace
  metadata:
   name: monitoring
  
  ---
  # prometheus-rbac.yaml
  apiVersion: v1
  kind: ServiceAccount
  metadata:
   name: prometheus
   namespace: monitoring
  
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: ClusterRole
  metadata:
   name: prometheus
  rules:
  - apiGroups: [""]
   resources:
   - nodes
   - nodes/proxy
   - services
   - endpoints
   - pods
   verbs: ["get", "list", "watch"]
  - apiGroups:
   - extensions
   resources:
   - ingresses
   verbs: ["get", "list", "watch"]
  - nonResourceURLs: ["/metrics"]
   verbs: ["get"]
  
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: ClusterRoleBinding
  metadata:
   name: prometheus
  roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: prometheus
  subjects:
  - kind: ServiceAccount
   name: prometheus
   namespace: monitoring
  
  ---
  # prometheus-config.yaml
  apiVersion: v1
  kind: ConfigMap
  metadata:
   name: prometheus-config
   namespace: monitoring
  data:
   prometheus.yml: |
   global:
   scrape_interval: 15s
   evaluation_interval: 15s
  
   rule_files:
   - "/etc/prometheus/rules/*.yml"
  
   alerting:
   alertmanagers:
   - static_configs:
   - targets:
   - alertmanager:9093
  
   scrape_configs:
   - job_name: 'prometheus'
   static_configs:
   - targets: ['localhost:9090']
  
   - job_name: 'kubernetes-apiservers'
   kubernetes_sd_configs:
   - role: endpoints
   scheme: https
   tls_config:
   ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   relabel_configs:
   - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
   action: keep
   regex: default;kubernetes;https
  
   - job_name: 'kubernetes-nodes'
   kubernetes_sd_configs:
   - role: node
   scheme: https
   tls_config:
   ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   relabel_configs:
   - action: labelmap
   regex: __meta_kubernetes_node_label_(.+)
   - target_label: __address__
   replacement: kubernetes.default.svc:443
   - source_labels: [__meta_kubernetes_node_name]
   regex: (.+)
   target_label: __metrics_path__
   replacement: /api/v1/nodes/${1}/proxy/metrics
  
   - job_name: 'kubernetes-cadvisor'
   kubernetes_sd_configs:
   - role: node
   scheme: https
   tls_config:
   ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   relabel_configs:
   - action: labelmap
   regex: __meta_kubernetes_node_label_(.+)
   - target_label: __address__
   replacement: kubernetes.default.svc:443
   - source_labels: [__meta_kubernetes_node_name]
   regex: (.+)
   target_label: __metrics_path__
   replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
  
   - job_name: 'kubernetes-service-endpoints'
   kubernetes_sd_configs:
   - role: endpoints
   relabel_configs:
   - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
   action: keep
   regex: true
   - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
   action: replace
   target_label: __metrics_path__
   regex: (.+)
   - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
   action: replace
   regex: ([^:]+)(?::\d+)?;(\d+)
   replacement: $1:$2
   target_label: __address__
   - action: labelmap
   regex: __meta_kubernetes_service_label_(.+)
   - source_labels: [__meta_kubernetes_namespace]
   action: replace
   target_label: kubernetes_namespace
   - source_labels: [__meta_kubernetes_service_name]
   action: replace
   target_label: kubernetes_name
  
   alert_rules.yml: |
   groups:
   - name: kubernetes-alerts
   rules:
   - alert: KubernetesPodCrashLooping
   expr: rate(kube_pod_container_status_restarts_total[5m]) * 60 * 5 > 0
   for: 2m
   labels:
   severity: warning
   annotations:
   summary: Pod {{ $labels.pod }} is crash looping
   description: Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} is restarting {{ $value }} times / 5 minutes.
  
   - alert: KubernetesPodNotReady
   expr: kube_pod_status_phase{phase="Pending",job="kube-state-metrics"} > 0
   for: 5m
   labels:
   severity: warning
   annotations:
   summary: Pod {{ $labels.pod }} not ready
   description: Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has been in a non-ready state for longer than 5 minutes.
  
   - alert: KubernetesMemoryPressure
   expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
   for: 2m
   labels:
   severity: critical
   annotations:
   summary: Node {{ $labels.node }} under memory pressure
   description: Node {{ $labels.node }} is under memory pressure.
  
   - alert: KubernetesDiskPressure
   expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
   for: 2m
   labels:
   severity: critical
   annotations:
   summary: Node {{ $labels.node }} under disk pressure
   description: Node {{ $labels.node }} is under disk pressure.
  
   - alert: KubernetesOutOfDisk
   expr: kube_node_status_condition{condition="OutOfDisk",status="true"} == 1
   for: 2m
   labels:
   severity: critical
   annotations:
   summary: Node {{ $labels.node }} out of disk
   description: Node {{ $labels.node }} has insufficient free disk space.
  
   - alert: KubernetesApiServerDown
   expr: up{job="kubernetes-apiservers"} == 0
   for: 5m
   labels:
   severity: critical
   annotations:
   summary: API server unreachable
   description: API server has been unreachable for more than 5 minutes.
  
  ---
  # prometheus-deployment.yaml
  apiVersion: apps/v1
  kind: Deployment
  metadata:
   name: prometheus
   namespace: monitoring
   labels:
   app: prometheus
  spec:
   replicas: 1
   selector:
   matchLabels:
   app: prometheus
   template:
   metadata:
   labels:
   app: prometheus
   spec:
   serviceAccountName: prometheus
   containers:
   - name: prometheus
   image: prom/prometheus:v2.40.0
   args:
   - '--config.file=/etc/prometheus/prometheus.yml'
   - '--storage.tsdb.path=/prometheus/'
   - '--web.console.libraries=/etc/prometheus/console_libraries'
   - '--web.console.templates=/etc/prometheus/consoles'
   - '--storage.tsdb.retention.time=200h'
   - '--web.enable-lifecycle'
   ports:
   - containerPort: 9090
   resources:
   requests:
   cpu: 500m
   memory: 500M
   limits:
   cpu: 1
   memory: 1Gi
   volumeMounts:
   - name: config-volume
   mountPath: /etc/prometheus/
   - name: prometheus-storage-volume
   mountPath: /prometheus/
   volumes:
   - name: config-volume
   configMap:
   defaultMode: 420
   name: prometheus-config
   - name: prometheus-storage-volume
   emptyDir: {}
  
  ---
  apiVersion: v1
  kind: Service
  metadata:
   name: prometheus-service
   namespace: monitoring
   annotations:
   prometheus.io/scrape: 'true'
   prometheus.io/port: '9090'
  spec:
   selector:
   app: prometheus
   type: NodePort
   ports:
   - port: 8080
   targetPort: 9090
   nodePort: 30000
  
  ---
  # alertmanager-config.yaml
  apiVersion: v1
  kind: ConfigMap
  metadata:
   name: alertmanager-config
   namespace: monitoring
  data:
   config.yml: |
   global:
   smtp_smarthost: 'localhost:587'
   smtp_from: 'alerts@yourcompany.com'
   slack_api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
  
   route:
   group_by: ['alertname']
   group_wait: 10s
   group_interval: 10s
   repeat_interval: 1h
   receiver: 'web.hook'
   routes:
   - match:
   severity: critical
   receiver: 'critical-alerts'
   - match:
   severity: warning
   receiver: 'warning-alerts'
  
   receivers:
   - name: 'web.hook'
   webhook_configs:
   - url: 'http://127.0.0.1:5001/'
  
   - name: 'critical-alerts'
   slack_configs:
   - api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
   channel: '#critical-alerts'
   title: 'Critical Alert'
   text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'
   send_resolved: true
   email_configs:
   - to: 'admin@yourcompany.com'
   subject: 'Critical Alert: {{ .GroupLabels.alertname }}'
   body: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'
  
   - name: 'warning-alerts'
   slack_configs:
   - api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
   channel: '#alerts'
   title: 'Warning Alert'
   text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'
   send_resolved: true
  
   inhibit_rules:
   - source_match:
   severity: 'critical'
   target_match:
   severity: 'warning'
   equal: ['alertname', 'dev', 'instance']
  
  ---
  # alertmanager-deployment.yaml
  apiVersion: apps/v1
  kind: Deployment
  metadata:
   name: alertmanager
   namespace: monitoring
  spec:
   replicas: 1
   selector:
   matchLabels:
   app: alertmanager
   template:
   metadata:
   name: alertmanager
   labels:
   app: alertmanager
   spec:
   containers:
   - name: alertmanager
   image: prom/alertmanager:v0.25.0
   args:
   - '--config.file=/etc/alertmanager/config.yml'
   - '--storage.path=/alertmanager'
   ports:
   - name: alertmanager
   containerPort: 9093
   resources:
   requests:
   cpu: 500m
   memory: 500M
   limits:
   cpu: 1
   memory: 1Gi
   volumeMounts:
   - name: config-volume
   mountPath: /etc/alertmanager
   - name: alertmanager-storage-volume
   mountPath: /alertmanager
   volumes:
   - name: config-volume
   configMap:
   name: alertmanager-config
   - name: alertmanager-storage-volume
   emptyDir: {}
  
  ---
  apiVersion: v1
  kind: Service
  metadata:
   name: alertmanager
   namespace: monitoring
  spec:
   selector:
   app: alertmanager
   type: NodePort
   ports:
   - port: 9093
   targetPort: 9093
   nodePort: 31000
  
  ---
  # grafana-config.yaml
  apiVersion: v1
  kind: ConfigMap
  metadata:
   name: grafana-datasources
   namespace: monitoring
  data:
   prometheus.yaml: |
   {
   "apiVersion": 1,
   "datasources": [
   {
   "access":"proxy",
   "editable": true,
   "name": "prometheus",
   "orgId": 1,
   "type": "prometheus",
   "url": "http://prometheus-service:8080",
   "version": 1
   }
   ]
   }
  
  ---
  # grafana-deployment.yaml
  apiVersion: apps/v1
  kind: Deployment
  metadata:
   name: grafana
   namespace: monitoring
  spec:
   replicas: 1
   selector:
   matchLabels:
   app: grafana
   template:
   metadata:
   name: grafana
   labels:
   app: grafana
   spec:
   containers:
   - name: grafana
   image: grafana/grafana:9.3.2
   ports:
   - name: grafana
   containerPort: 3000
   resources:
   limits:
   memory: "1Gi"
   cpu: "1000m"
   requests: 
   memory: 500M
   cpu: "500m"
   volumeMounts:
   - mountPath: /var/lib/grafana
   name: grafana-storage
   - mountPath: /etc/grafana/provisioning/datasources
   name: grafana-datasources
   readOnly: false
   volumes:
   - name: grafana-storage
   emptyDir: {}
   - name: grafana-datasources
   configMap:
   defaultMode: 420
   name: grafana-datasources
  
  ---
  apiVersion: v1
  kind: Service
  metadata:
   name: grafana
   namespace: monitoring
   annotations:
   prometheus.io/scrape: 'true'
   prometheus.io/port: '3000'
  spec:
   selector: 
   app: grafana
   type: NodePort 
   ports:
   - port: 3000
   targetPort: 3000
   nodePort: 32000
  
  ---
  # kube-state-metrics-deployment.yaml
  apiVersion: apps/v1
  kind: Deployment
  metadata:
   name: kube-state-metrics
   namespace: monitoring
   labels:
   app: kube-state-metrics
  spec:
   selector:
   matchLabels:
   app: kube-state-metrics
   replicas: 1
   template:
   metadata:
   labels:
   app: kube-state-metrics
   spec:
   serviceAccountName: kube-state-metrics
   containers:
   - name: kube-state-metrics
   image: k8s.gcr.io/kube-state-metrics/kube-state-metrics:v2.7.0
   ports:
   - name: http-metrics
   containerPort: 8080
   - name: telemetry
   containerPort: 8081
   readinessProbe:
   httpGet:
   path: /
   port: 8081
   initialDelaySeconds: 5
   timeoutSeconds: 5
   livenessProbe:
   httpGet:
   path: /healthz
   port: 8080
   initialDelaySeconds: 5
   timeoutSeconds: 5
   resources:
   requests:
   memory: 70Mi
   cpu: 5m
  
  ---
  apiVersion: v1
  kind: ServiceAccount
  metadata:
   name: kube-state-metrics
   namespace: monitoring
  
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: ClusterRole
  metadata:
   name: kube-state-metrics
  rules:
  - apiGroups: [""]
   resources:
   - configmaps
   - secrets
   - nodes
   - pods
   - services
   - resourcequotas
   - replicationcontrollers
   - limitranges
   - persistentvolumeclaims
   - persistentvolumes
   - namespaces
   - endpoints
   verbs: ["list", "watch"]
  - apiGroups: ["apps"]
   resources:
   - statefulsets
   - daemonsets
   - deployments
   - replicasets
   verbs: ["list", "watch"]
  - apiGroups: ["batch"]
   resources:
   - cronjobs
   - jobs
   verbs: ["list", "watch"]
  - apiGroups: ["autoscaling"]
   resources:
   - horizontalpodautoscalers
   verbs: ["list", "watch"]
  - apiGroups: ["authentication.k8s.io"]
   resources:
   - tokenreviews
   verbs: ["create"]
  - apiGroups: ["authorization.k8s.io"]
   resources:
   - subjectaccessreviews
   verbs: ["create"]
  - apiGroups: ["policy"]
   resources:
   - poddisruptionbudgets
   verbs: ["list", "watch"]
  - apiGroups: ["certificates.k8s.io"]
   resources:
   - certificatesigningrequests
   verbs: ["list", "watch"]
  - apiGroups: ["storage.k8s.io"]
   resources:
   - storageclasses
   - volumeattachments
   verbs: ["list", "watch"]
  - apiGroups: ["admissionregistration.k8s.io"]
   resources:
   - mutatingwebhookconfigurations
   - validatingwebhookconfigurations
   verbs: ["list", "watch"]
  - apiGroups: ["networking.k8s.io"]
   resources:
   - networkpolicies
   - ingresses
   verbs: ["list", "watch"]
  - apiGroups: ["coordination.k8s.io"]
   resources:
   - leases
   verbs: ["list", "watch"]
  
  ---
  apiVersion: rbac.authorization.k8s.io/v1
  kind: ClusterRoleBinding
  metadata:
   name: kube-state-metrics
  roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: kube-state-metrics
  subjects:
  - kind: ServiceAccount
   name: kube-state-metrics
   namespace: monitoring
  
  ---
  apiVersion: v1
  kind: Service
  metadata:
   name: kube-state-metrics
   namespace: monitoring
   labels:
   app: kube-state-metrics
   annotations:
   prometheus.io/scrape: 'true'
  spec:
   ports:
   - name: http-metrics
   port: 8080
   targetPort: http-metrics
   protocol: TCP
   - name: telemetry
   port: 8081
   targetPort: telemetry
   protocol: TCP
   selector:
   app: kube-state-metrics
  ```
  
  This monitoring setup provides:
  - Prometheus for metrics collection and alerting
  - Grafana for visualization and dashboards
  - AlertManager for alert routing and notification
  - Kube-state-metrics for Kubernetes metrics
  - Comprehensive alerting rules
  - Service discovery for automatic target detection
  - RBAC permissions for secure access