kubernetes部署metric server
6 min read

kubernetes部署metric server

检查环境
[root@k8s-master-01 ~]# kubectl get nodes
NAME           STATUS   ROLES    AGE   VERSION
192.168.9.28   Ready    <none>   12h   v1.15.10
192.168.9.29   Ready    <none>   12h   v1.15.10
[root@k8s-master-01 ~]# kubectl get cs
NAME                 STATUS    MESSAGE              ERROR
scheduler            Healthy   ok
controller-manager   Healthy   ok
etcd-1               Healthy   {"health": "true"}
etcd-2               Healthy   {"health": "true"}
etcd-0               Healthy   {"health": "true"}
[root@k8s-master-01 ~]# kubectl get node -o wide
NAME           STATUS   ROLES    AGE   VERSION    INTERNAL-IP    EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION          CONTAINER-RUNTIME
192.168.9.28   Ready    <none>   12h   v1.15.10   192.168.9.28   <none>        CentOS Linux 7 (Core)   3.10.0-693.el7.x86_64   docker://18.3.1
192.168.9.29   Ready    <none>   12h   v1.15.10   192.168.9.29   <none>        CentOS Linux 7 (Core)   3.10.0-693.el7.x86_64   docker://18.3.1

由于没有安装metric server,导致查看node/pod的资源使用情况,执行都报错

[root@k8s-master-01 ~]# kubectl top nodes
Error from server (NotFound): the server could not find the requested resource (get services http:heapster:)
[root@k8s-master-01 ~]# kubectl top pods
Error from server (NotFound): the server could not find the requested resource (get services http:heapster:)
下载

直接用curl下载这个yaml文件进行安装

kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.7/components.yaml

需提前将gcr.io/k8s-staging-metrics-server/metrics-server:master这个镜像给拉取下来。
下面附上两个镜像的tar包,直接下载下来,然后doker load < 包名 即可。
k8s.gcr.io/metrics-server/metrics-server:v0.3.7
registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0

[root@k8s-master-01 deployment]# docker images
REPOSITORY                                                        TAG                 IMAGE ID            CREATED             SIZE
k8s.gcr.io/metrics-server/metrics-server                          v0.3.7              07c9e703ca2c        6 months ago        55.4MB
registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64   3.0                 99e59f495ffa        4 years ago         747kB

注意:将这两个镜像推送到node节点。

修改kube-apiserver配置文件

部署metric server需要在kube-apiserver的配置文件中添加如下配置:

  --requestheader-allowed-names= \
  --requestheader-extra-headers-prefix=X-Remote-Extra- \
  --requestheader-group-headers=X-Remote-Group \
  --requestheader-username-headers=X-Remote-User \
  --proxy-client-cert-file={{ var_ssl_k8s_dir }}/{{ var_ssl_aggregator_cert_prefix }}.pem \
  --proxy-client-key-file={{ var_ssl_k8s_dir }}/{{ var_ssl_aggregator_cert_prefix }}-key.pem \
  --enable-aggregator-routing=true

注意:

  • requestheader-allowed-names:如果不为空的情况下,需要保证此设定值与证书中的CN一致。
  • enable-aggregator-routing:master节点上没有kube-proxy时,需要设定为true。

问题故障解决:https://www.cnblogs.com/Dev0ps/p/10778328.html
生成metric-server证书

创建一个metric-csr.json文件
# cat metric-csr.json
{
    "CN": "metrics-server",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
生成metric-server证书
# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes metric-csr.json | cfssljson -bare metric-server
将证书放到ssl目录
# cp metric-server.pem /opt/kubernetes/ssl/
# cp metric-server-key.pem /opt/kubernetes/ssl/
# ll
总用量 40
-rw-------. 1 root root 1675 11月  2 18:49 admin-key.pem
-rw-r--r--. 1 root root 1399 11月  2 18:49 admin.pem
-rw-------. 1 root root 1679 11月  2 18:49 ca-key.pem
-rw-r--r--. 1 root root 1359 11月  2 18:49 ca.pem
-rw-------. 1 root root 1679 11月  2 18:49 kube-proxy-key.pem
-rw-r--r--. 1 root root 1403 11月  2 18:49 kube-proxy.pem
-rw-------. 1 root root 1675 11月  3 15:33 metric-server-key.pem
-rw-r--r--. 1 root root 1399 11月  3 15:33 metric-server.pem
-rw-------. 1 root root 1679 11月  2 18:49 server-key.pem
-rw-r--r--. 1 root root 1627 11月  2 18:49 server.pem
修改kube-apiserver配置文件
# cat /opt/kubernetes/cfg/kube-apiserver
KUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--etcd-servers=https://192.168.9.27:2379,https://192.168.9.28:2379,https://192.168.9.29:2379 \
--insecure-bind-address=127.0.0.1 \
--bind-address=192.168.9.27 \
--insecure-port=8080 \
--secure-port=6443 \
--advertise-address=192.168.9.27 \
--allow-privileged=true \
--service-cluster-ip-range=10.10.10.0/24 \
--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--kubelet-https=true \
--enable-bootstrap-token-auth \
--token-auth-file=/opt/kubernetes/cfg/token.csv \
--service-node-port-range=30000-50000 \
--tls-cert-file=/opt/kubernetes/ssl/server.pem  \
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/opt/kubernetes/ssl/ca.pem \
--etcd-certfile=/opt/kubernetes/ssl/server.pem \
--etcd-keyfile=/opt/kubernetes/ssl/server-key.pem \
--requestheader-client-ca-file=/opt/kubernetes/ssl/ca.pem \
--requestheader-allowed-names=  \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--proxy-client-cert-file=/opt/kubernetes/ssl/metric-server.pem \
--proxy-client-key-file=/opt/kubernetes/ssl/metric-server-key.pem \
--enable-aggregator-routing=true"
重启kube-apiserver服务
# systemctl restart kube-apiserver
# systemctl status  kube-apiserver
# tail -f /var/log/message
启动metric-server
# cd /opt/kubernetes/deployment/metrics/
[root@k8s-master-01 metrics]# ll
总用量 8
-rw-r--r--. 1 root root 3439 11月  3 14:28 components.yaml
[root@k8s-master-01 metrics]# kubectl apply -f components.yaml
clusterrole.rbac.authorization.k8s.io/system:aggregated-metrics-reader created
clusterrolebinding.rbac.authorization.k8s.io/metrics-server:system:auth-delegator created
rolebinding.rbac.authorization.k8s.io/metrics-server-auth-reader created
apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io created
serviceaccount/metrics-server created
deployment.apps/metrics-server created
service/metrics-server created
clusterrole.rbac.authorization.k8s.io/system:metrics-server created
clusterrolebinding.rbac.authorization.k8s.io/system:metrics-server created
检查
[root@k8s-master-01 metrics]# kubectl get rs -n monitor
NAME                        DESIRED   CURRENT   READY   AGE
metrics-server-7c96fc4888   1         1         1       13m
[root@k8s-master-01 metrics]# kubectl get deployment -n monitor
NAME             READY   UP-TO-DATE   AVAILABLE   AGE
metrics-server   1/1     1            1           13m
[root@k8s-master-01 metrics]# kubectl get service -n monitor
NAME             TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)   AGE
metrics-server   ClusterIP   10.10.10.89   <none>        443/TCP   13m
[root@k8s-master-01 metrics]# kubectl get pod -n monitor
NAME                              READY   STATUS    RESTARTS   AGE
metrics-server-7c96fc4888-zvnjq   1/1     Running   0          13m
[root@k8s-master-01 metrics]# kubectl top pod -n monitor
NAME                              CPU(cores)   MEMORY(bytes)
metrics-server-7c96fc4888-zvnjq   1m           13Mi
[root@k8s-master-01 metrics]# kubectl top nodes
NAME           CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%
192.168.9.28   63m          3%     944Mi           25%
192.168.9.29   84m          4%     893Mi           24%

附上components.yaml文件内容

[root@k8s-master-01 metrics]# cat components.yaml
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:aggregated-metrics-reader
  labels:
    rbac.authorization.k8s.io/aggregate-to-view: "true"
    rbac.authorization.k8s.io/aggregate-to-edit: "true"
    rbac.authorization.k8s.io/aggregate-to-admin: "true"
rules:
- apiGroups: ["metrics.k8s.io"]
  resources: ["pods", "nodes"]
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: metrics-server:system:auth-delegator
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: monitor
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: metrics-server-auth-reader
  namespace: monitor
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: monitor
---
apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
  name: v1beta1.metrics.k8s.io
spec:
  service:
    name: metrics-server
    namespace: monitor
  group: metrics.k8s.io
  version: v1beta1
  insecureSkipTLSVerify: true
  groupPriorityMinimum: 100
  versionPriority: 100
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: metrics-server
  namespace: monitor
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: metrics-server
  namespace: monitor
  labels:
    k8s-app: metrics-server
spec:
  selector:
    matchLabels:
      k8s-app: metrics-server
  template:
    metadata:
      name: metrics-server
      labels:
        k8s-app: metrics-server
    spec:
      serviceAccountName: metrics-server
      volumes:
      # mount in tmp so we can safely use from-scratch images and/or read-only containers
      - name: tmp-dir
        emptyDir: {}
      containers:
      - name: metrics-server
        image: k8s.gcr.io/metrics-server/metrics-server:v0.3.7
        imagePullPolicy: IfNotPresent
        command:
        - /metrics-server
        - --kubelet-insecure-tls
        - --kubelet-preferred-address-types=InternalIP
        args:
          - --cert-dir=/tmp
          - --secure-port=4443
        ports:
        - name: main-port
          containerPort: 4443
          protocol: TCP
        securityContext:
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 1000
        volumeMounts:
        - name: tmp-dir
          mountPath: /tmp
      nodeSelector:
        kubernetes.io/os: linux
---
apiVersion: v1
kind: Service
metadata:
  name: metrics-server
  namespace: monitor
  labels:
    kubernetes.io/name: "Metrics-server"
    kubernetes.io/cluster-service: "true"
spec:
  selector:
    k8s-app: metrics-server
  ports:
  - port: 443
    protocol: TCP
    targetPort: main-port
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:metrics-server
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - nodes
  - nodes/stats
  - namespaces
  - configmaps
  verbs:
  - get
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:metrics-server
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:metrics-server
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: monitor
京ICP备19055754号