k8s1.27版本集群部署minio分布式

需求：

1.创建4个pv，一个pv一个minio-pod。使用sts动态分配pvc(根据存储类找到pv)。----持久化

2.暴露minio的9001端口。（nodeport）----管理界面

镜像：minio/minio:RELEASE.2023-03-20T20-16-18Z--->换国内源

说明：本环境仅测试，两个minio的pod放在master节点，在master加了污点容忍，其他两个miniopod放在工作节点，上生产环境需要调整，官方建议：4个minio需要4块硬盘，2个完好可读，3个完好可读写，可以挂一块磁盘到pv目录。

159m  2个minio+/data/minio-pv-0,1

160n   1个minio+/data/minio-pv-2

161n   1个minio+/data/minio-pv-3

159m# mkdir /data/minio-pv-{0..1} && chmod 777 -R /data/minio-pv-{0..1}

160n#mkdir /data/minio-pv-2 && chmod 777 -R /data/minio-pv-2

161n#mkdir /data/minio-pv-3 && chmod 777 -R /data/minio-pv-3

#159master上操作

mkdir minio-kind && cd minio-kind

vim minio-ns.yaml  #官方建议独立分配minio命名空间

apiVersion: v1
kind: Namespace
metadata:name: minio

vim minio-pv.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: minio-storage   #创建类
provisioner: kubernetes.io/no-provisioner
---
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-pv-0labels:type: local
spec:capacity:storage: 1Gi  #生产环境需要调整扩大accessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: minio-storagelocal:path: /data/minio-pv-0nodeAffinity:required:nodeSelectorTerms:- matchExpressions:- key: kubernetes.io/hostnameoperator: Invalues:- 159m     #kubectl get nodes查看node名称
---
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-pv-1labels:type: local
spec:capacity:storage: 1GiaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: minio-storagelocal:path: /data/minio-pv-1nodeAffinity:required:nodeSelectorTerms:- matchExpressions:- key: kubernetes.io/hostnameoperator: Invalues:- 159m
---
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-pv-2labels:type: local
spec:capacity:storage: 1GiaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: minio-storagelocal:path: /data/minio-pv-2nodeAffinity:required:nodeSelectorTerms:- matchExpressions:- key: kubernetes.io/hostnameoperator: Invalues:- 160n
---
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-pv-3labels:type: local
spec:capacity:storage: 1GiaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: minio-storagelocal:path: /data/minio-pv-3nodeAffinity:required:nodeSelectorTerms:- matchExpressions:- key: kubernetes.io/hostnameoperator: Invalues:- 161n

vim minio-sts.yaml  #svc,secret一起写了，注意要保证集群的coredns正常工作

apiVersion: apps/v1
kind: StatefulSet
metadata:name: minionamespace: minio
spec:serviceName: "minio"replicas: 4  # 集群节点数量selector:matchLabels:app: miniotemplate:metadata:labels:app: miniospec:tolerations:- key: "node-role.kubernetes.io/control-plane"operator: "Exists"effect: "NoSchedule"containers:- name: minio#image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/quay.io/minio/minio:latestimage: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/minio/minio:RELEASE.2023-03-20T20-16-18Zargs:- server- http://minio-0.minio:9000/data- http://minio-1.minio:9000/data- http://minio-2.minio:9000/data- http://minio-3.minio:9000/data- --console-address- ":9001"  # 控制台地址参数- --address- ":9000"  # API 地址参数ports:- containerPort: 9000name: minio- containerPort: 9001name: minio-consoleenv:- name: MINIO_ROOT_PASSWORDvalueFrom:secretKeyRef:name: minio-secretkey: MINIO_ROOT_PASSWORD- name: MINIO_ROOT_USERvalueFrom:secretKeyRef:name: minio-secretkey: MINIO_ROOT_USER#- name: MINIO_BROWSER#value: "on"volumeMounts:- name: minio-datamountPath: /datavolumeClaimTemplates:- metadata:name: minio-dataspec:accessModes: [ "ReadWriteOnce" ]storageClassName: minio-storage  # 使用 minio-storage 存储类resources:requests:storage: 1Gi  # 每个节点的存储大小
---
apiVersion: v1
kind: Service
metadata:name: minionamespace: minio
spec:#clusterIP: Noneports:- port: 9000targetPort: 9000protocol: TCPname: minionodePort: 30001- port: 9001targetPort: 9001protocol: TCPname: minio-consolenodePort: 30002selector:app: miniotype: NodePort
---
apiVersion: v1
kind: Secret
metadata:name: minio-secretnamespace: minio
type: Opaque
data: MINIO_ROOT_USER: cXdlYXNkMTE=  #用户名base64，这里的参数根据版本不同可能需要调整为accessMINIO_ROOT_PASSWORD: cXdlYXNkMTE=  #密码base64

running起来了过一会再用mc(linux系统需要安装)连接测试，初始化时间较长

#测试连接

kubectl port-forward svc/minio 9000:9000 -n minio #临时测试
mc alias set myminio http://127.0.0.1:9000 用户名 密码

#防火墙开放30001和30002端口，提供外部访问(nodeport)

ufw allow 30001/TCP

ufw allow 30002/TCP

#访问管理页面

10.10.10.159:30002 

报错：

ERROR Unable to initialize backend: format.json file: expected format-type: fs, found: xl

delete -f .资源后，需要清理干净pv残留的数据，隐藏目录需要指定删除，不然会一直引用之前的数据。这里删除的是我的pv环境目录，根据场景删除。

rm -fr /data/minio-pv-0/.minio.sys/

rm -fr /data/minio-pv-1/.minio.sys/

rm -fr /data/minio-pv-2/.minio.sys/

rm -fr /data/minio-pv-3/.minio.sys/