使用 Go 在 Kubernetes 中構建自己的准入控制器

語言: CN / TW / HK

C T O

   

 

Go Rust Python Istio containerd CoreDNS Envoy etcd Fluentd Harbor Helm Jaeger Kubernetes Open Policy Agent Prometheus Rook TiKV TUF Vitess Arg o Buildpacks CloudEvents CNI Contour Cortex CRI-O Falco Flux gRPC KubeEdge Linkerd NATS Notary OpenTracing Operator Framework SPIFFE SPIRE     Thanos

使用 Go 在 Kubernetes 中構建自己的准入控制器

准入控制與 RBAC

使用基於角色的訪問控制 ( RBAC ),您可以限制對 K8s 資源的訪問並施加不同型別的限制,例如允許或拒絕具有特定角色的人建立、列出、更新和刪除任何型別的物件,或者還可以限制訪問名稱空間本身。但如果你想要更多呢?例如,如果建立了部署並且您想要驗證它的名稱應該有字首名稱,或者它的映象名稱應該只允許來自私有倉庫?… RBAC 無法驗證和確保這一點!所以,這就是准入控制器的用武之地。

什麼是准入控制器?

正如 Kubernetes 文件中提到的:“准入控制器是一段程式碼,它在物件持久化之前擷取對 Kubernetes API 伺服器的請求,但在請求經過身份驗證和授權之後。” 因此,基本上它有助於定義規則或安全措施來強制執行叢集的使用方式。

有兩個准入控制器可供使用:

  • MutatingAdmissionWebhook : 允許使用 mutating webhook 在資源被持久化之前修改它的內容。
  • ValidatingAdmissionWebhooks :允許使用驗證 webhook 來執行自定義准入策略。

我們可以配置這些 webhook 以訪問包含我們准入控制器服務邏輯的 webhook 伺服器。在這篇文章中,我開發了一個簡單的 webhook 伺服器,其中包括變異和驗證部署的開頭與特定的字首。 prod

讓我們開始吧!

我們需要什麼來開發我們自己的准入控制器?

在開始實施之前,讓我們先來概述一下這個過程。首先,您需要編寫和部署 webhook 伺服器並使其可訪問。接下來,在 K8s 上配置 webhook 以訪問您的 webhook 伺服器。之後, Webhook POST 使用 JSON 有效負載作為 AdmissionReviewAPI 物件傳送請求,其中包含有關請求的所有詳細資訊。然後, webhook 應該得到一個 JSON 有效負載 AdmissionReview 響應,其中包含關於是否允許請求的決定。

現在實施:

1、部署 Webhook 伺服器

package main

import (
 "flag"
 "fmt"
 "io/ioutil"
 "net/http"
 "strings"

 "github.com/rs/zerolog/log"
 admission "k8s.io/api/admission/v1"
 appsv1 "k8s.io/api/apps/v1"
 corev1 "k8s.io/api/core/v1"
 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 "k8s.io/apimachinery/pkg/runtime"
 "k8s.io/apimachinery/pkg/runtime/serializer"
 "k8s.io/kubernetes/pkg/apis/apps/v1"

 "encoding/json"
)

var (
 runtimeScheme = runtime.NewScheme()
 codecFactory  = serializer.NewCodecFactory(runtimeScheme)
 deserializer  = codecFactory.UniversalDeserializer()
)

// add kind AdmissionReview in scheme
func init() {
 _ = corev1.AddToScheme(runtimeScheme)
 _ = admission.AddToScheme(runtimeScheme)
 _ = v1.AddToScheme(runtimeScheme)
}

type admitv1Func func(admission.AdmissionReview) *admission.AdmissionResponse

type admitHandler struct {
 v1 admitv1Func
}

func AdmitHandler(f admitv1Func) admitHandler {
 return admitHandler{
  v1: f,
 }
}

// serve handles the http portion of a request prior to handing to an admit
// function
func serve(w http.ResponseWriter, r *http.Request, admit admitHandler) {
 var body []byte
 if r.Body != nil {
  if data, err := ioutil.ReadAll(r.Body); err == nil {
   body = data
  }
 }

 // verify the content type is accurate
 contentType := r.Header.Get("Content-Type")
 if contentType != "application/json" {
  log.Error().Msgf("contentType=%s, expect application/json", contentType)
  return
 }

 log.Info().Msgf("handling request: %s", body)
 var responseObj runtime.Object
 if obj, gvk, err := deserializer.Decode(body, nil, nil); err != nil {
  msg := fmt.Sprintf("Request could not be decoded: %v", err)
  log.Error().Msg(msg)
  http.Error(w, msg, http.StatusBadRequest)
  return

 } else {
  requestedAdmissionReview, ok := obj.(*admission.AdmissionReview)
  if !ok {
   log.Error().Msgf("Expected v1.AdmissionReview but got: %T", obj)
   return
  }
  responseAdmissionReview := &admission.AdmissionReview{}
  responseAdmissionReview.SetGroupVersionKind(*gvk)
  responseAdmissionReview.Response = admit.v1(*requestedAdmissionReview)
  responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID
  responseObj = responseAdmissionReview

 }
 log.Info().Msgf("sending response: %v", responseObj)
 respBytes, err := json.Marshal(responseObj)
 if err != nil {
  log.Err(err)
  http.Error(w, err.Error(), http.StatusInternalServerError)
  return
 }
 w.Header().Set("Content-Type", "application/json")
 if _, err := w.Write(respBytes); err != nil {
  log.Err(err)
 }
}

func serveMutate(w http.ResponseWriter, r *http.Request) {
 serve(w, r, AdmitHandler(mutate))
}
func serveValidate(w http.ResponseWriter, r *http.Request) {
 serve(w, r, AdmitHandler(validate))
}

// adds prefix 'prod' to every incoming Deployment, example: prod-apps
func mutate(ar admission.AdmissionReview) *admission.AdmissionResponse {
 log.Info().Msgf("mutating deployments")
 deploymentResource := metav1.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}
 if ar.Request.Resource != deploymentResource {
  log.Error().Msgf("expect resource to be %s", deploymentResource)
  return nil
 }
 raw := ar.Request.Object.Raw
 deployment := appsv1.Deployment{}

 if _, _, err := deserializer.Decode(raw, nil, &deployment); err != nil {
  log.Err(err)
  return &admission.AdmissionResponse{
   Result: &metav1.Status{
    Message: err.Error(),
   },
  }
 }
 newDeploymentName := fmt.Sprintf("prod-%s", deployment.GetName())
 pt := admission.PatchTypeJSONPatch
 deploymentPatch := fmt.Sprintf(`[{ "op": "add", "path": "/metadata/name", "value": "%s" }]`, newDeploymentName)
 return &admission.AdmissionResponse{Allowed: true, PatchType: &pt, Patch: []byte(deploymentPatch)}
}

// verify if a Deployment has the 'prod' prefix name
func validate(ar admission.AdmissionReview) *admission.AdmissionResponse {
 log.Info().Msgf("validating deployments")
 deploymentResource := metav1.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}
 if ar.Request.Resource != deploymentResource {
  log.Error().Msgf("expect resource to be %s", deploymentResource)
  return nil
 }
 raw := ar.Request.Object.Raw
 deployment := appsv1.Deployment{}
 if _, _, err := deserializer.Decode(raw, nil, &deployment); err != nil {
  log.Err(err)
  return &admission.AdmissionResponse{
   Result: &metav1.Status{
    Message: err.Error(),
   },
  }
 }
 if !strings.HasPrefix(deployment.GetName(), "prod-") {
  return &admission.AdmissionResponse{
   Allowed: false, Result: &metav1.Status{
    Message: "Deployment's prefix name \"prod\" not found",
   },
  }
 }
 return &admission.AdmissionResponse{Allowed: true}
}

func main() {
 var tlsKey, tlsCert string
 flag.StringVar(&tlsKey, "tlsKey", "/etc/certs/tls.key", "Path to the TLS key")
 flag.StringVar(&tlsCert, "tlsCert", "/etc/certs/tls.crt", "Path to the TLS certificate")
 flag.Parse()
 http.HandleFunc("/mutate", serveMutate)
 http.HandleFunc("/validate", serveValidate)
 log.Info().Msg("Server started ...")
 log.Fatal().Err(http.ListenAndServeTLS(":8443", tlsCert, tlsKey, nil)).Msg("webhook server exited")
}

在上面的程式碼中,它處理 /validate 來自 /mutate 兩個呼叫的請求: validate mutate . 該 mutate 呼叫通過將部署字首名稱新增到部署 :white_check_mark: 的 JSON 補丁操作獲取部署的名稱和響應。然後, validate 接收驗證 webhook 請求,並在程式碼示例中驗證部署的字首名稱是否有效, prod 並接受或拒絕該請求。

我作為物件部署在 Kubernetes 叢集中, deployment 並建立了一個服務作為 webhook 伺服器的前端(您也可以將 webhook 伺服器部署在叢集之外)

apiVersion: apps/v1
kind: Deployment
metadata:
  name: webhook-server
  namespace: production
  labels:
    app: webhook-server
spec:
  replicas: 1
  selector:
    matchLabels:
      app: webhook-server
  template:
    metadata:
      labels:
        app: webhook-server
    spec:
      containers:
        - name: webhook-server
          image: bashayralabdullah/webhook-server:v1.0
          imagePullPolicy: Always
          ports:
            - containerPort: 8443
          volumeMounts:
            - name: tls-certs
              mountPath: /etc/certs
              readOnly: true
      volumes:
        - name: tls-certs
          secret:
            secretName: webhook-server-tls
---
apiVersion: v1
kind: Service
metadata:
  name: webhook-server
  namespace: production
spec:
  selector:
    app: webhook-server
  ports:
    - port: 443
      targetPort: 8443

由於 API 伺服器僅通過 HTTPS admission webhook 伺服器進行通訊,因此它需要 TLS 證書的 CA 資訊。因此,我建立了自簽名證書,然後建立了 kubernetes.io/tls 用於儲存證書及其關聯金鑰的金鑰:

> kubectl create secret tls webhook-server-tls \
    --cert "certs/tls.crt" \
    --key "certs/tls.key" -n production

然後 webhook_server.yml K8s 中應用:

> kubectl create -f webhook_server.yml

注意:在我的 GitHub 儲存庫中,您將找到 create_k8s_objects.sh 指令碼,該指令碼將建立自簽名證書、 tls secret webhook 伺服器和 webhook

https://github.com/Bashayr29/k8s-admission-controller

2.配置准入Webhook

注意:要驗證 admissionregistration.k8s.io/v1 admissionregistration.k8s.io/v1beta1 啟用: kubectl api-versions | grep -i admissionregistration.k8s.io 。請檢查先決條件以確保准入控制器已啟用。

先決條件: https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites

為了註冊 admission webhooks ,建立 MutatingWebhookConfiguration ValidatingWebhookConfigurationAPI 物件:

註冊 admission webhooks : https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#webhook-configuration

---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
  name: deployment-validation
webhooks:
  - name: "deployment-validation.production.svc"
    namespaceSelector:
      matchExpressions:
        - key: kubernetes.io/metadata.name
          operator: In
          values: [ "production" ]
    rules:
      - operations: [ "CREATE"]
        apiGroups: [ "apps" ]
        apiVersions: [ "v1" ]
        resources: [ "deployments" ]
        scope: "Namespaced"
    clientConfig:
      service:
        namespace: production
        name: webhook-server
        path: "/validate"
      caBundle: <ENCODED_CA>
    admissionReviewVersions: ["v1"]
    sideEffects: None
    timeoutSeconds: 5

---
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
  name: deployment-mutation
webhooks:
  - name: "deployment-mutation.production.svc"
    namespaceSelector:
      matchExpressions:
        - key: kubernetes.io/metadata.name
          operator: In
          values: [ "production" ]
    rules:
      - operations: [ "CREATE"]
        apiGroups: [ "apps" ]
        apiVersions: [ "v1" ]
        resources: [ "deployments" ]
        scope: "Namespaced"
    clientConfig:
      service:
        namespace: production
        name: webhook-server
        path: "/mutate"
      caBundle: <ENCODED_CA>
    admissionReviewVersions: ["v1"]
    sideEffects: None
    timeoutSeconds: 5

如果您的   webhook 伺服器部署在 K8s 之外,您需要在兩個 webhook 中替換 service block under clientConfigby url ,如下所示:

apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
  name: validator
webhooks:
  - name: admission-server.production.svc
    clientConfig:
      url: <YOUR_EXTERNAL_URL>/validate
    rules:
      ...

K8s 中應用

kubectl apply -f webhooks.yml

3. 測試:nail_care:

現在我們的控制器已經寫好了,看看所有建立的資源:

> kubectl get deployments.apps,svc,secret -n production
NAME                             READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/webhook-server   1/1     1            1           36m
NAME                     TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)   AGE
service/webhook-server   ClusterIP   10.245.197.198   <none>        443/TCP   76m
NAME                         TYPE                                  DATA   AGE
secret/webhook-server-tls    kubernetes.io/tls                     2      59m
> kubectl get mutatingwebhookconfigurations.admissionregistration.k8s.io
NAME                  WEBHOOKS   AGE
deployment-mutation   1          37m
> kubectl get validatingwebhookconfigurations.admissionregistration.k8s.io
NAME                    WEBHOOKS   AGE
deployment-validation   1          38m

讓我們建立一個簡單的 Nginx 部署來測試我們的 webhook 伺服器:

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
  namespace: production
  labels:
    app: nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
        - name: nginx
          image: nginx:1.14.2
          ports:
            - containerPort: 80

在建立 Nginx 部署時檢視 webhook-server 的日誌:

kubectl logs -n production webhook-server-5999c6d74d-4k4pg --follow

執行它

> kubectl create -f test_deployment.yml

你可以看到日誌內容如下

{"level":"info","time":"2022-05-04T03:18:36Z","message":"Server started ..."}
{"level":"info","time":"2022-05-04T03:19:03Z","message":"handling request: {\"kind\":\"AdmissionReview\",\"apiVersion\":\"admission.k8s.io/v1\",\"request\":{\"uid\":\"8cde5fab-
...
{"level":"info","time":"2022-05-04T03:19:03Z","message":"mutating deployments"}
{"level":"info","time":"2022-05-04T03:19:03Z","message":"sending response: &AdmissionReview{Request:nil,Response:&AdmissionResponse{UID:8cde5fab-8dd5-449d-8f92-455c005b1f3b,Allowed:true,Result:nil,Patch:*[91 123 32 34 111 112 34 58 32 34 97 100 100 34 44 32 34 112 97 116 104 34 58 32 34 47 109 101 116 97 100 97 116 97 47 110 97 109 101 34 44 32 34 118 97 108 117 101 34 58 32 34 112 114 111 100 45 110 103 105 110 120 45 100 101 112 108 111 121 109 101 110 116 34 32 125 93],PatchType:*JSONPatch,AuditAnnotations:map[string]string{},Warnings:[],},}"}
...
{"level":"info","time":"2022-05-04T03:19:03Z","message":"validating deployments"}
{"level":"info","time":"2022-05-04T03:19:03Z","message":"sending response: &AdmissionReview{Request:nil,Response:&AdmissionResponse{UID:60773af9-ae64-41de-ade3-45f3424b59bf,Allowed:true,Result:nil,Patch:nil,PatchType:nil,AuditAnnotations:map[string]string{},Warnings:[],},}"}

如果我們得到 Nginx 部署的名稱,我們可以看到它已從 更改 nginx-deployment prod-nginx-deployment

> kubectl get deployments.apps -n production 
NAME READY UP-TO-DATE AVAILABLE AGE 
prod-nginx-deployment 1/1 1 1 5m24s

結束

在這裡,我們介紹瞭如何在 Golang 中建立一個簡單的 Kubernetes 准入控制器並將其部署到 K8s 叢集。這很容易,您可以考慮使用許多不同的情況作為準入控制器,通過應用一些策略使您的叢集更加安全。

我希望這可以幫到你!