[root@localhost prometheus]# kubectl version
Client Version: version.Info{Major:"1", Minor:"12", GitVersion:"v1.12.5", GitCommit:"51dd616cdd25d6ee22c83a858773b607328a18ec", GitTreeState:"clean", BuildDate:"2019-01-16T18:24:45Z", GoVersion:"go1.10.7", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"12", GitVersion:"v1.12.5", GitCommit:"51dd616cdd25d6ee22c83a858773b607328a18ec", GitTreeState:"clean", BuildDate:"2019-01-16T18:14:49Z", GoVersion:"go1.10.7", Compiler:"gc", Platform:"linux/amd64"}
prom/prometheus:v2.4.3
grafana/grafana:5.3.4
Then I deploy the grafana-kubernetes-app plugin, here is the configmap of prometheus:
[root@localhost prometheus]# cat prometheus-cm.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-config
namespace: monitoring
data:
prometheus.yml: |
global:
scrape_interval: 15s
scrape_timeout: 15s
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['localhost:9090']
- job_name: 'kubernetes-nodes'
kubernetes_sd_configs:
- role: node
relabel_configs:
- source_labels: [__address__]
regex: '(.*):10250'
replacement: '${1}:9100'
target_label: __address__
action: replace
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- job_name: 'kubernetes-kubelet'
kubernetes_sd_configs:
- role: node
relabel_configs:
- source_labels: [__address__]
regex: '(.*):10250'
replacement: '${1}:10255'
target_label: __address__
action: replace
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- job_name: 'kubernetes-cadvisor'
kubernetes_sd_configs:
- role: node
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
- job_name: 'kubernetes-apiservers'
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: default;kubernetes;https
- job_name: 'kubernetes-service-endpoints'
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
[root@localhost prometheus]# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-779ffd89bd-rvd42 1/1 Running 0 3h29m
kube-state-metrics-fcdc7d964-qmbfg 1/1 Running 0 87m
kubernetes-dashboard-659798bd99-j6rj2 1/1 Running 0 3h26m
node-exporter-bhvvt 1/1 Running 0 87m
node-exporter-jrncb 1/1 Running 0 87m
node-exporter-q8tzh 1/1 Running 0 87m
node-exporter-r52fg 1/1 Running 0 87m
[root@localhost prometheus]# kubectl get pod -n monitoring
NAME READY STATUS RESTARTS AGE
grafana-c7bf74c8-sbmmp 1/1 Running 0 93m
grafana-chown-rd96q 0/1 Completed 0 99m
prometheus-7c958795dd-6ndfj 1/1 Running 0 106m
Only the k8s-container dashboard has data,I also see that the kube-state-metrics pod has the following exceptions:
E0204 09:33:42.538610 1 reflector.go:205] k8s.io/kube-state-metrics/collectors/pod.go:187: Failed to list *v1.Pod: pods is forbidden: User "system:serviceaccount:kube-system:default" cannot list resource "pods" in API group "" at the cluster scope
E0204 09:33:42.738156 1 reflector.go:205] k8s.io/kube-state-metrics/collectors/persistentvolumeclaim.go:78: Failed to list *v1.PersistentVolumeClaim: persistentvolumeclaims is forbidden: User "system:serviceaccount:kube-system:default" cannot list resource "persistentvolumeclaims" in API group "" at the cluster scope
E0204 09:33:42.923342 1 reflector.go:205] k8s.io/kube-state-metrics/collectors/daemonset.go:82: Failed to list *v1beta1.DaemonSet: daemonsets.extensions is forbidden: User "system:serviceaccount:kube-system:default" cannot list resource "daemonsets" in API group "extensions" at the cluster scope
E0204 09:33:42.924153 1 reflector.go:205] k8s.io/kube-state-metrics/collectors/cronjob.go:93: Failed to list *v2alpha1.CronJob: cronjobs.batch is forbidden: User "system:serviceaccount:kube-system:default" cannot list resource "cronjobs" in API group "batch" at the cluster scope
E0204 09:33:42.925186 1 reflector.go:205] k8s.io/kube-state-metrics/collectors/replicaset.go:87: Failed to list *v1beta1.ReplicaSet: replicasets.extensions is forbidden: User "system:serviceaccount:kube-system:default" cannot list resource "replicasets" in API group "extensions" at the cluster scope
E0204 09:33:42.937971 1 reflector.go:205] k8s.io/kube-state-metrics/collectors/namespace.go:80: Failed to list *v1.Namespace: namespaces is forbidden: User "system:serviceaccount:kube-system:default" cannot list resource "namespaces" in API group "" at the cluster scope
E0204 09:33:43.138574 1 reflector.go:205] k8s.io/kube-state-metrics/collectors/node.go:142: Failed to list *v1.Node: nodes is forbidden: User "system:serviceaccount:kube-system:default" cannot list resource "nodes" in API group "" at the cluster scope
god.. who can help me.
Thanks.