01-监控体系

$ git clone https://github.com/kubernetes-incubator/metrics-server
$ cd metrics-server
$ kubectl create -f deploy/1.8+/

# metrics.k8s.io这个API会出现在kubernetes API列表中

https://<apiserver_ip>/apis/custom-metrics.metrics.k8s.io/v1beta1/namespaces/default/pods/sample-metrics-app/http_requests

# 当访问这个URL的时候，Custom Metrics APIServer就会去Prometheus里查询
# sample-metrics-app这个Pod的http_requests指标的值，然后按照固定的格式返回给访问者

# 1.部署Prometheus项目。使用Prometheus Operator来完成
$ kubectl apply -f demos/monitoring/prometheus-operator.yaml
clusterrole "prometheus-operator" created
serviceaccount "prometheus-operator" created
clusterrolebinding "prometheus-operator" created
deployment "prometheus-operator" created

$ kubectl apply -f demos/monitoring/sample-prometheus-instance.yaml
clusterrole "prometheus" created
serviceaccount "prometheus" created
clusterrolebinding "prometheus" created
prometheus "sample-metrics-prom" created
service "sample-metrics-prom" created

# 2.把Custom Metrics APIServer部署起来
$ kubectl apply -f demos/monitoring/custom-metrics.yaml
namespace "custom-metrics" created
serviceaccount "custom-metrics-apiserver" created
clusterrolebinding "custom-metrics:system:auth-delegator" created
rolebinding "custom-metrics-auth-reader" created
clusterrole "custom-metrics-read" created
clusterrolebinding "custom-metrics-read" created
deployment "custom-metrics-apiserver" created
service "api" created
apiservice "v1beta1.custom-metrics.metrics.k8s.io" created
clusterrole "custom-metrics-server-resources" created
clusterrolebinding "hpa-controller-custom-metrics" created

# 3.为CustomMetrics APIServer创建对应的ClusterRoleBinding，以便能够使用curl来直接访问Custom Metrics的API
$ kubectl create clusterrolebinding allowall-cm --clusterrole custom-metrics-server-resources --user system:anonymous
clusterrolebinding "allowall-cm" created

# 4.把待监控的应用和HPA部署起来
$ kubectl apply -f demos/monitoring/sample-metrics-app.yaml
deployment "sample-metrics-app" created
service "sample-metrics-app" created
servicemonitor "sample-metrics-app" created     # 这个是Prometheus Operator用来指定被监控Pod的一个配置文件
horizontalpod
scaler "sample-metrics-app-hpa" created
ingress "sample-metrics-app" created

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: sample-metrics-app
  labels:
    service-monitor: sample-metrics-app
spec:
  selector:
    matchLabels:
      app: sample-metrics-app
  endpoints:
  - port: web

# 通过Label Selector为Prometheus指定被监控的应用

apiVersion:
scaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
  name: sample-metrics-app-hpa
spec:
  scaleTargetRef:       # 指定被监控的对象，包括API、类型、名字
    apiVersion: apps/v1
    kind: Deployment
    name: sample-metrics-app
  minReplicas: 2        # 指定被监控对象的最小实例数目
  maxReplicas: 10       # 指定被监控对象的最大实例数目
  metrics:              # 指定这个HPA进行Scale的依据
  - type: Object
    object:             # 获取这个Metrics的途径，包括类型和名字
      target:
        kind: Service
        name: sample-metrics-app
      metricName: http_requests     # Metrics的名字
      targetValue: 100

# HPA的配置，就是设置Auto Scaling规则的地方

https://<apiserver_ip>/apis/custom-metrics.metrics.k8s.io/v1beta1/namespaces/default/services/sample-metrics-app/http_requests

# 这个URL对应的被监控对象，是应用对应的Service，这与Pod的Custom Metrics URL不同
# 对于一个多实例应用来说，通过Service来采集Pod的Custom Metrics才是合理的做法

# 通过hey测试工具来对应用增加一些访问压力：
$ # Install hey
$ docker run -it -v /usr/local/bin:/go/bin golang:1.8 go get github.com/rakyll/hey

$ export APP_ENDPOINT=$(kubectl get svc sample-metrics-app -o template --template {{.spec.clusterIP}}); echo ${APP_ENDPOINT}
$ hey -n 50000 -c 1000 http://${APP_ENDPOINT}

# 此时访问Service的Custom Metrics URL，就能看到这个URL已经可以返回应用收到的HTTP请求的数量
$ curl -sSLk https://<apiserver_ip>/apis/custom-metrics.metrics.k8s.io/v1beta1/namespaces/default/services/sample-metrics-app/http_requests
{
  "kind": "MetricValueList",
  "apiVersion": "custom-metrics.metrics.k8s.io/v1beta1",
  "metadata": {
    "selfLink": "/apis/custom-metrics.metrics.k8s.io/v1beta1/namespaces/default/services/sample-metrics-app/http_requests"
  },
  "items": [
    {
      "describedObject": {
        "kind": "Service",
        "name": "sample-metrics-app",
        "apiVersion": "/__internal"
      },
      "metricName": "http_requests",
      "timestamp": "2018-11-30T20:56:34Z",
      "value": "501484m"
    }
  ]
}

  if (request.url == "/metrics") {
    response.end(
      "# HELP http_requests_total The amount of requests served by the server in total\n"+
      "# TYPE http_requests_total counter\nhttp_requests_total " + totalrequests + "\n");
    return;
  }

// 应用的/metrics对应的HTTP response里返回的，是http_request_total的值，也是Prometheus收集到的值。