feat(k8s): add Azure Blob Storage CSI driver configuration and resources

Signed-off-by: 孙振宇 <>

cluster/manifests/freeleaps-devops-system/jenkins/pv.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: jenkins-pv
+  namespace: freeleaps-devops-system
+spec:
+  storageClassName: freeleaps-node-local
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 20Gi
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+    path: /mnt/data/jenkins

cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure-cloud-provider-secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: azure-cloud-provider
+  namespace: freeleaps-storage-system
+type: Opaque
+data:
+  cloud-config: ewogICJjbG91ZCI6ICJBenVyZVB1YmxpY0Nsb3VkIiwKICAidGVuYW50SWQiOiAiY2YxNTFlZTgtNWMyYy00ZmU3LWExYzQtODA5YmE0M2M5ZjI0IiwKICAic3Vic2NyaXB0aW9uSWQiOiAiMGEyODAwNjgtZGVjNC00YmYwLTlmMDQtNjViNjRmNDEyYjUwIiwKICAicmVzb3VyY2VHcm91cCI6ICJrOHMiLAogICJsb2NhdGlvbiI6ICJ3ZXN0dXMyIiwKICAiYWFkQ2xpZW50SWQiOiAiN2NkMWRmMTktMjRlYS00NmQ3LWFjZDMtNTMzNjI4MzEzOWUwIiwKICAiYWFkQ2xpZW50U2VjcmV0IjogIjJWVThRflBDTXFFYmgtelpKTFlhRUVQQml6NTJJQzVieVp4c2liZlYiLAogICJ1c2VNYW5hZ2VkSWRlbnRpdHlFeHRlbnNpb24iOiBmYWxzZSwKICAidXNlckFzc2lnbmVkSWRlbnRpdHlJRCI6ICIiLAogICJ1c2VJbnN0YW5jZU1ldGFkYXRhIjogdHJ1ZQp9Cg==

cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure-json-base64.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+# check if the azure.json not exist
+if [ ! -f azure.json ]; then
+  echo "azure.json file not found, exiting"
+  exit 1
+fi
+
+cat azure.json | base64 | awk '{printf $0}'; echo

cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure.json

@@ -0,0 +1,12 @@
+{
+  "cloud": "AzurePublicCloud",
+  "tenantId": "cf151ee8-5c2c-4fe7-a1c4-809ba43c9f24",
+  "subscriptionId": "0a280068-dec4-4bf0-9f04-65b64f412b50",
+  "resourceGroup": "k8s",
+  "location": "westus2",
+  "aadClientId": "7cd1df19-24ea-46d7-acd3-5336283139e0",
+  "aadClientSecret": "2VU8Q~PCMqEbh-zZJLYaEEPBiz52IC5byZxsibfV",
+  "useManagedIdentityExtension": false,
+  "userAssignedIdentityID": "",
+  "useInstanceMetadata": true
+}

cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/storage-classes.yaml

@@ -0,0 +1,268 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-fuse-2-std-lrs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Standard_LRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+  protocol: fuse2
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-fuse-2-premium-lrs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Premium_LRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+  protocol: fuse2
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-fuse-2-standard-grs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Standard_GRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+  protocol: fuse2
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-fuse-2-standard-ragrs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Standard_RAGRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+  protocol: fuse2
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-fuse-2-standard-zrs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Standard_ZRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+  protocol: fuse2
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-fuse-2-premium-zrs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Premium_ZRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+  protocol: fuse2
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-nfs
+provisioner: blob.csi.azure.com
+parameters:
+  protocol: nfs
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-std-lrs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Premium_LRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-premium-lrs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Premium_LRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-standard-grs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Standard_GRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-standard-ragrs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Standard_RAGRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-standard-zrs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Standard_ZRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-blob-premium-zrs
+provisioner: blob.csi.azure.com
+parameters:
+  skuName: Premium_ZRS  # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
+mountOptions:
+  - -o allow_other
+  - --file-cache-timeout-in-seconds=120
+  - --use-attr-cache=true
+  - --cancel-list-on-mount-seconds=10  # prevent billing charges on mounting
+  - -o attr_timeout=120
+  - -o entry_timeout=120
+  - -o negative_timeout=120
+  - --log-level=LOG_WARNING  # LOG_WARNING, LOG_INFO, LOG_DEBUG
+  - --cache-size-mb=1000  # Default will be 80% of available memory, eviction will happen beyond that.

cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/values.yaml

@@ -0,0 +1,186 @@
+image:
+  baseRepo: mcr.microsoft.com
+  blob:
+    repository: /k8s/csi/blob-csi
+    tag: latest
+    pullPolicy: IfNotPresent
+  csiProvisioner:
+    repository: /oss/kubernetes-csi/csi-provisioner
+    tag: v5.1.0
+    pullPolicy: IfNotPresent
+  livenessProbe:
+    repository: /oss/kubernetes-csi/livenessprobe
+    tag: v2.14.0
+    pullPolicy: IfNotPresent
+  nodeDriverRegistrar:
+    repository: /oss/kubernetes-csi/csi-node-driver-registrar
+    tag: v2.12.0
+    pullPolicy: IfNotPresent
+  csiResizer:
+    repository: /oss/kubernetes-csi/csi-resizer
+    tag: v1.12.0
+    pullPolicy: IfNotPresent
+
+cloud: AzurePublicCloud
+
+## Reference to one or more secrets to be used when pulling images
+## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+imagePullSecrets: []
+# - name: myRegistryKeySecretName
+
+serviceAccount:
+  create: true # When true, service accounts will be created for you. Set to false if you want to use your own.
+  controller: csi-blob-controller-sa # Name of Service Account to be created or used
+  node: csi-blob-node-sa # Name of Service Account to be created or used
+
+rbac:
+  create: true
+  name: blob
+
+## Collection of annotations to add to all the pods
+podAnnotations: {}
+## Collection of labels to add to all the pods
+podLabels: {}
+# -- Custom labels to add into metadata
+customLabels: {}
+  # k8s-app: blob-csi-driver
+
+## Leverage a PriorityClass to ensure your pods survive resource shortages
+## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+priorityClassName: system-cluster-critical
+## Security context give the opportunity to run container as nonroot by setting a securityContext
+## by example :
+## securityContext: { runAsUser: 1001 }
+securityContext: {}
+
+controller:
+  name: csi-blob-controller
+  cloudConfigSecretName: azure-cloud-provider
+  cloudConfigSecretNamespace: freeleaps-storage-system
+  allowEmptyCloudConfig: true
+  hostNetwork: true # this setting could be disabled if controller does not depend on MSI setting
+  metricsPort: 29634
+  livenessProbe:
+    healthPort: 29632
+  replicas: 2
+  runOnMaster: false
+  runOnControlPlane: true
+  logLevel: 5
+  resources:
+    csiProvisioner:
+      limits:
+        memory: 500Mi
+      requests:
+        cpu: 10m
+        memory: 20Mi
+    livenessProbe:
+      limits:
+        memory: 100Mi
+      requests:
+        cpu: 10m
+        memory: 20Mi
+    blob:
+      limits:
+        memory: 800Mi
+      requests:
+        cpu: 10m
+        memory: 20Mi
+    csiResizer:
+      limits:
+        memory: 500Mi
+      requests:
+        cpu: 10m
+        memory: 20Mi
+  affinity: {}
+  nodeSelector: {}
+  tolerations:
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/controlplane"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/control-plane"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "CriticalAddonsOnly"
+      operator: "Exists"
+      effect: "NoSchedule"
+
+node:
+  name: csi-blob-node
+  cloudConfigSecretName: azure-cloud-provider
+  cloudConfigSecretNamespace: freeleaps-storage-system
+  allowEmptyCloudConfig: true
+  allowInlineVolumeKeyAccessWithIdentity: false
+  maxUnavailable: 1
+  metricsPort: 29635
+  livenessProbe:
+    healthPort: 29633
+  logLevel: 5
+  enableBlobfuseProxy: true
+  blobfuseProxy:
+    installBlobfuse: false
+    blobfuseVersion: "1.4.5"
+    installBlobfuse2: true
+    blobfuse2Version: "2.4.0"
+    setMaxOpenFileNum: true
+    maxOpenFileNum: "9000000"
+    disableUpdateDB: true
+    migrateK8sRepo: false
+    setReadAheadSize: true
+  blobfuseCachePath: /mnt
+  appendTimeStampInCacheDir: false
+  mountPermissions: 0777
+  resources:
+    livenessProbe:
+      limits:
+        memory: 100Mi
+      requests:
+        cpu: 10m
+        memory: 20Mi
+    nodeDriverRegistrar:
+      limits:
+        memory: 100Mi
+      requests:
+        cpu: 10m
+        memory: 20Mi
+    blob:
+      limits:
+        memory: 2100Mi
+      requests:
+        cpu: 10m
+        memory: 20Mi
+    aznfswatchdog:
+      limits:
+        memory: 100Mi
+      requests:
+        cpu: 10m
+        memory: 20Mi
+  affinity: {}
+  nodeSelector: {}
+  tolerations:
+    - operator: "Exists"
+  enableAznfsMount: true
+
+feature:
+  fsGroupPolicy: ReadWriteOnceWithFSType
+  enableGetVolumeStats: false
+
+driver:
+  name: blob.csi.azure.com
+  customUserAgent: ""
+  userAgentSuffix: "OSS-helm"
+  azureGoSDKLogLevel: "INFO" # available values: ""(no logs), DEBUG, INFO, WARNING, ERROR
+  httpsProxy: ""
+  httpProxy: ""
+
+linux:
+  kubelet: /var/lib/kubelet
+  distro: debian
+
+workloadIdentity:
+  clientID: ""
+  # [optional] If the AAD application or user-assigned managed identity is not in the same tenant as the cluster
+  # then set tenantID with the application or user-assigned managed identity tenant ID
+  tenantID: ""