溫馨提示×
您好，登錄后才能下訂單哦！
點擊 登錄注冊 即表示同意《億速云用戶服務條款》
您好，登錄后才能下訂單哦！
Kubernetes部署（一）：架構及功能說明
Kubernetes部署（二）：系統環境初始化
Kubernetes部署（三）：CA證書制作
Kubernetes部署（四）：ETCD集群部署
Kubernetes部署（五）：Haproxy、Keppalived部署
Kubernetes部署（六）：Master節點部署
Kubernetes部署（七）：Node節點部署
Kubernetes部署（八）：Flannel網絡部署
Kubernetes部署（九）：CoreDNS、Dashboard、Ingress部署
Kubernetes部署（十）：儲存之glusterfs和heketi部署
Kubernetes部署（十一）：管理之Helm和Rancher部署
Kubernetes部署（十二）：helm部署harbor企業級鏡像倉庫
本指南支持在Kubernetes集群中集成，部署和管理GlusterFS容器化存儲節點。這使Kubernetes管理員能夠為其用戶提供可靠的共享存儲。
包括設置指南、其中包含一個示例服務器pod，它使用動態配置的GlusterFS卷進行存儲。對于那些希望測試或了解有關此主題的更多信息的人，請按照主要自述文件中的快速入門說明 了解gluster-kubernetes
本指南旨在演示Heketi在Kubernetes環境中管理Gluster的最小示例。
#使用file -s 查看硬盤如果顯示為data則為原始塊設備。如果不是data類型，可先用pvcreate，pvremove來變更。
[root@node-04 ~]# file -s /dev/sdc
/dev/sdc: x86 boot sector, code offset 0xb8
[root@node-04 ~]# pvcreate /dev/sdc
WARNING: dos signature detected on /dev/sdc at offset 510. Wipe it? [y/n]: y
Wiping dos signature on /dev/sdc.
Physical volume "/dev/sdc" successfully created.
[root@node-04 ~]# pvremove /dev/sdc
Labels on physical volume "/dev/sdc" successfully wiped.
[root@node-04 ~]# file -s /dev/sdc
/dev/sdc: datayum install -y glusterfs-client glusterfs-fuse socatmodprobe dm_thin_poolHeketi提供CLI，為用戶提供管理Kubernetes中GlusterFS的部署和配置的方法。 在您的客戶端計算機上下載并安裝下載并安裝heketi-cli，下載的heketi-cli版本最好是和heketi服務端版本一致，不然可能會出現報錯。
部署GlusterFS DaemonSet
{
"kind": "DaemonSet",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "glusterfs",
"labels": {
"glusterfs": "deployment"
},
"annotations": {
"description": "GlusterFS Daemon Set",
"tags": "glusterfs"
}
},
"spec": {
"template": {
"metadata": {
"name": "glusterfs",
"labels": {
"glusterfs-node": "daemonset"
}
},
"spec": {
"nodeSelector": {
"storagenode" : "glusterfs"
},
"hostNetwork": true,
"containers": [
{
"image": "gluster/gluster-centos:latest",
"imagePullPolicy": "Always",
"name": "glusterfs",
"volumeMounts": [
{
"name": "glusterfs-heketi",
"mountPath": "/var/lib/heketi"
},
{
"name": "glusterfs-run",
"mountPath": "/run"
},
{
"name": "glusterfs-lvm",
"mountPath": "/run/lvm"
},
{
"name": "glusterfs-etc",
"mountPath": "/etc/glusterfs"
},
{
"name": "glusterfs-logs",
"mountPath": "/var/log/glusterfs"
},
{
"name": "glusterfs-config",
"mountPath": "/var/lib/glusterd"
},
{
"name": "glusterfs-dev",
"mountPath": "/dev"
},
{
"name": "glusterfs-cgroup",
"mountPath": "/sys/fs/cgroup"
}
],
"securityContext": {
"capabilities": {},
"privileged": true
},
"readinessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 60,
"exec": {
"command": [
"/bin/bash",
"-c",
"systemctl status glusterd.service"
]
}
},
"livenessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 60,
"exec": {
"command": [
"/bin/bash",
"-c",
"systemctl status glusterd.service"
]
}
}
}
],
"volumes": [
{
"name": "glusterfs-heketi",
"hostPath": {
"path": "/var/lib/heketi"
}
},
{
"name": "glusterfs-run"
},
{
"name": "glusterfs-lvm",
"hostPath": {
"path": "/run/lvm"
}
},
{
"name": "glusterfs-etc",
"hostPath": {
"path": "/etc/glusterfs"
}
},
{
"name": "glusterfs-logs",
"hostPath": {
"path": "/var/log/glusterfs"
}
},
{
"name": "glusterfs-config",
"hostPath": {
"path": "/var/lib/glusterd"
}
},
{
"name": "glusterfs-dev",
"hostPath": {
"path": "/dev"
}
},
{
"name": "glusterfs-cgroup",
"hostPath": {
"path": "/sys/fs/cgroup"
}
}
]
}
}
}
}$ kubectl create -f glusterfs-daemonset.json$ kubectl get nodesstoragenode=glusterfs在該節點上設置標簽，將gluster容器部署到指定節點上。[root@node-01 heketi]# kubectl label node 10.31.90.204 storagenode=glusterfs
[root@node-01 heketi]# kubectl label node 10.31.90.205 storagenode=glusterfs
[root@node-01 heketi]# kubectl label node 10.31.90.206 storagenode=glusterfs根據需要,驗證pod正在節點上運行，至少應運行三個pod。
$ kubectl get pods接下來我們將為Heketi創建一個ServiceAccount：
{
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": {
"name": "heketi-service-account"
}
}$ kubectl create -f heketi-service-account.json我們現在必須建立該服務帳戶控制gluster pod的能力。我們通過為新創建的服務帳戶創建集群角色綁定來實現此目的。
$ kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account{
"_port_comment": "Heketi Server Port Number",
"port": "8080",
"_use_auth": "Enable JWT authorization. Please enable for deployment",
"use_auth": false,
"_jwt": "Private keys for access",
"jwt": {
"_admin": "Admin has access to all APIs",
"admin": {
"key": "My Secret"
},
"_user": "User only has access to /volumes endpoint",
"user": {
"key": "My Secret"
}
},
"_glusterfs_comment": "GlusterFS Configuration",
"glusterfs": {
"_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh",
"executor": "kubernetes",
"_db_comment": "Database file name",
"db": "/var/lib/heketi/heketi.db",
"kubeexec": {
"rebalance_on_expansion": true
},
"sshexec": {
"rebalance_on_expansion": true,
"keyfile": "/etc/heketi/private_key",
"fstab": "/etc/fstab",
"port": "22",
"user": "root",
"sudo": false
}
},
"_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.",
"backup_db_to_kube_secret": false
}
$ kubectl create secret generic heketi-config-secret --from-file=./heketi.json{
"kind": "List",
"apiVersion": "v1",
"items": [
{
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": "deploy-heketi",
"labels": {
"glusterfs": "heketi-service",
"deploy-heketi": "support"
},
"annotations": {
"description": "Exposes Heketi Service"
}
},
"spec": {
"selector": {
"name": "deploy-heketi"
},
"ports": [
{
"name": "deploy-heketi",
"port": 8080,
"targetPort": 8080
}
]
}
},
{
"kind": "Deployment",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "deploy-heketi",
"labels": {
"glusterfs": "heketi-deployment",
"deploy-heketi": "deployment"
},
"annotations": {
"description": "Defines how to deploy Heketi"
}
},
"spec": {
"replicas": 1,
"template": {
"metadata": {
"name": "deploy-heketi",
"labels": {
"name": "deploy-heketi",
"glusterfs": "heketi-pod",
"deploy-heketi": "pod"
}
},
"spec": {
"serviceAccountName": "heketi-service-account",
"containers": [
{
"image": "heketi/heketi:8",
"imagePullPolicy": "Always",
"name": "deploy-heketi",
"env": [
{
"name": "HEKETI_EXECUTOR",
"value": "kubernetes"
},
{
"name": "HEKETI_DB_PATH",
"value": "/var/lib/heketi/heketi.db"
},
{
"name": "HEKETI_FSTAB",
"value": "/var/lib/heketi/fstab"
},
{
"name": "HEKETI_SNAPSHOT_LIMIT",
"value": "14"
},
{
"name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
"value": "y"
}
],
"ports": [
{
"containerPort": 8080
}
],
"volumeMounts": [
{
"name": "db",
"mountPath": "/var/lib/heketi"
},
{
"name": "config",
"mountPath": "/etc/heketi"
}
],
"readinessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 3,
"httpGet": {
"path": "/hello",
"port": 8080
}
},
"livenessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 30,
"httpGet": {
"path": "/hello",
"port": 8080
}
}
}
],
"volumes": [
{
"name": "db"
},
{
"name": "config",
"secret": {
"secretName": "heketi-config-secret"
}
}
]
}
}
}
}
]
}# kubectl create -f heketi-bootstrap.json
service "deploy-heketi" created
deployment "deploy-heketi" created
[root@node-01 heketi]# kubectl get pod
NAME READY STATUS RESTARTS AGE
deploy-heketi-8888799fd-cmfp6 1/1 Running 0 6m
glusterfs-7t5ls 1/1 Running 0 8m
glusterfs-drsx9 1/1 Running 0 8m
glusterfs-pnnn8 1/1 Running 0 8mkubectl port-forward deploy-heketi-8888799fd-cmfp6 :8080如果在運行命令的系統上本地端口8080空閑，則可以運行port-forward命令，以便它為了方便而綁定到8080：
kubectl port-forward deploy-heketi-8888799fd-cmfp6 18080:8080現在通過對Heketi服務運行示例查詢來驗證端口轉發是否正常工作。該命令應該打印將要轉發的本地端口。將其合并到URL中以測試服務，如下所示：
curl http://localhost:18080/hello
Handling connection for 18080
Hello from Heketi最后，為Heketi CLI客戶端設置環境變量，以便它知道如何到達Heketi Server。
export HEKETI_CLI_SERVER=http://localhost:18080接下來，我們將向Heketi提供有關要管理的GlusterFS集群的信息。我們通過拓撲文件提供此信息 。您克隆的repo中有一個示例拓撲文件，名為topology-sample.json。拓撲指定運行GlusterFS容器的Kubernetes節點以及每個節點的相應原始塊設備。
確保hostnames/manage指向下面顯示的確切名稱kubectl get nodes，并且hostnames/storage是存儲網絡的IP地址。
修改拓撲文件以反映您所做的選擇，然后部署它，如下所示：
{
"clusters": [
{
"nodes": [
{
"node": {
"hostnames": {
"manage": [
"10.31.90.204"
],
"storage":[
"10.31.90.204"
]
},
"zone": 1
},
"devices": [
"/dev/sdc"
]
},
{
"node": {
"hostnames": {
"manage": [
"10.31.90.205"
],
"storage":[
"10.31.90.205"
]
},
"zone": 1
},
"devices": [
"/dev/sdc"
]
},
{
"node": {
"hostnames": {
"manage": [
"10.31.90.206"
],
"storage":[
"10.31.90.206"
]
},
"zone": 1
},
"devices": [
"/dev/sdc"
]
}
]
}
]
}
[root@node-01 ~]# heketi-cli topology load --json=top.json
Creating cluster ... ID: e758afb77ee26d5f969d7efee1516e64
Allowing file volumes on cluster.
Allowing block volumes on cluster.
Creating node 10.31.90.204 ... ID: a6eedd58c118dcfe44a0db2af1a4f863
Adding device /dev/sdc ... OK
Creating node 10.31.90.205 ... ID: 4066962c14bcdebd28aca193b5690792
Adding device /dev/sdc ... OK
Creating node 10.31.90.206 ... ID: 91e42a2361f0266ae334354e5c34ce11
Adding device /dev/sdc ... OK執行此命令后會生成一個heketi-storage.json的文件，我們最好是將此文件里的"image": "heketi/heketi:dev"
改為"image": "heketi/heketi:8"
# heketi-client/bin/heketi-cli setup-openshift-heketi-storage然后在創建heketi相關服務
# kubectl create -f heketi-storage.json陷阱：如果heketi-cli在運行setup-openshift-heketi-storage子命令時報告“無空間”錯誤，則可能無意中運行topology load了服務器和heketi-cli的不匹配版本。停止正在運行的Heketi pod（kubectl scale deployment deploy-heketi --replicas=0），手動從存儲塊設備中刪除任何簽名，然后繼續運行Heketi pod（kubectl scale deployment deploy-heketi --replicas=1）。然后使用匹配版本的heketi-cli重新加載拓撲并重試該步驟。
# kubectl delete all,service,jobs,deployment,secret --selector="deploy-heketi"{
"kind": "List",
"apiVersion": "v1",
"items": [
{
"kind": "Secret",
"apiVersion": "v1",
"metadata": {
"name": "heketi-db-backup",
"labels": {
"glusterfs": "heketi-db",
"heketi": "db"
}
},
"data": {
},
"type": "Opaque"
},
{
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": "heketi",
"labels": {
"glusterfs": "heketi-service",
"deploy-heketi": "support"
},
"annotations": {
"description": "Exposes Heketi Service"
}
},
"spec": {
"selector": {
"name": "heketi"
},
"ports": [
{
"name": "heketi",
"port": 8080,
"targetPort": 8080
}
]
}
},
{
"kind": "Deployment",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "heketi",
"labels": {
"glusterfs": "heketi-deployment"
},
"annotations": {
"description": "Defines how to deploy Heketi"
}
},
"spec": {
"replicas": 1,
"template": {
"metadata": {
"name": "heketi",
"labels": {
"name": "heketi",
"glusterfs": "heketi-pod"
}
},
"spec": {
"serviceAccountName": "heketi-service-account",
"containers": [
{
"image": "heketi/heketi:8",
"imagePullPolicy": "Always",
"name": "heketi",
"env": [
{
"name": "HEKETI_EXECUTOR",
"value": "kubernetes"
},
{
"name": "HEKETI_DB_PATH",
"value": "/var/lib/heketi/heketi.db"
},
{
"name": "HEKETI_FSTAB",
"value": "/var/lib/heketi/fstab"
},
{
"name": "HEKETI_SNAPSHOT_LIMIT",
"value": "14"
},
{
"name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
"value": "y"
}
],
"ports": [
{
"containerPort": 8080
}
],
"volumeMounts": [
{
"mountPath": "/backupdb",
"name": "heketi-db-secret"
},
{
"name": "db",
"mountPath": "/var/lib/heketi"
},
{
"name": "config",
"mountPath": "/etc/heketi"
}
],
"readinessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 3,
"httpGet": {
"path": "/hello",
"port": 8080
}
},
"livenessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 30,
"httpGet": {
"path": "/hello",
"port": 8080
}
}
}
],
"volumes": [
{
"name": "db",
"glusterfs": {
"endpoints": "heketi-storage-endpoints",
"path": "heketidbstorage"
}
},
{
"name": "heketi-db-secret",
"secret": {
"secretName": "heketi-db-backup"
}
},
{
"name": "config",
"secret": {
"secretName": "heketi-config-secret"
}
}
]
}
}
}
}
]
}# kubectl create -f heketi-deployment.json
service "heketi" created
deployment "heketi" created使用諸如heketi-cli cluster list和之類的命令heketi-cli volume list 來確認先前建立的集群是否存在，以及Heketi是否知道在引導階段創建的db存儲卷。
heketi.cnlinux.club的A記錄解析為10.31.90.200。
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: ingress-heketi
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /
kubernetes.io/ingress.class: nginx
spec:
rules:
- host: heketi.cnlinux.club
http:
paths:
- path:
backend:
serviceName: heketi
servicePort: 8080[root@node-01 heketi]# kubectl create -f ingress-heketi.yaml在瀏覽器訪問http://heketi.cnlinux.club/hello
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: gluster-heketi
provisioner: kubernetes.io/glusterfs
parameters:
resturl: "http://heketi.cnlinux.club"
restauthenabled: "false"
volumetype: "replicate:2"[root@node-01 heketi]# kubectl create -f storageclass-gluster-heketi.yaml
[root@node-01 heketi]# kubectl get sc
NAME PROVISIONER AGE
gluster-heketi kubernetes.io/glusterfs 10sapiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: pvc-gluster-heketi
spec:
storageClassName: gluster-heketi
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi[root@node-01 heketi]# kubectl create -f pvc-gluster-heketi.yaml
[root@node-01 heketi]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc-gluster-heketi Bound pvc-d978f524-0b74-11e9-875c-005056826470 1Gi RWO gluster-heketi 30sapiVersion: v1
kind: Pod
metadata:
name: pod-pvc
spec:
containers:
- name: pod-pvc
image: busybox:latest
command:
- sleep
- "3600"
volumeMounts:
- name: gluster-volume
mountPath: "/pv-data"
volumes:
- name: gluster-volume
persistentVolumeClaim:
claimName: pvc-gluster-heketi[root@node-01 heketi]# kubectl create -f pod-pvc.yaml 進入容器查看是否已經掛載成功
[root@node-01 heketi]# kubectl exec pod-pvc -it /bin/sh
/ # df -h
Filesystem Size Used Available Use% Mounted on
overlay 47.8G 4.3G 43.5G 9% /
tmpfs 64.0M 0 64.0M 0% /dev
tmpfs 1.9G 0 1.9G 0% /sys/fs/cgroup
10.31.90.204:vol_675cc9fe0e959157919c886ea7786d33
1014.0M 42.7M 971.3M 4% /pv-data
/dev/sda3 47.8G 4.3G 43.5G 9% /dev/termination-log
/dev/sda3 47.8G 4.3G 43.5G 9% /etc/resolv.conf
/dev/sda3 47.8G 4.3G 43.5G 9% /etc/hostname
/dev/sda3 47.8G 4.3G 43.5G 9% /etc/hosts#往/pv-data寫文件，當容量超過1G時就自動退出了，證明容量限制是生效的。
/ # cd /pv-data/
/pv-data # dd if=/dev/zero of=/pv-data/test.img bs=8M count=300
123+0 records in
122+0 records out
1030225920 bytes (982.5MB) copied, 24.255925 seconds, 40.5MB/s在宿主機磁盤里查看是否創建了test.img文件
[root@node-04 cfg]# mount /dev/vg_2631413b8b87bbd6cb526568ab697d37/brick_1691ef862dd504e12e8384af76e5a9f2 /mnt
[root@node-04 cfg]# ll -h /mnt/brick/
total 982M
-rw-r--r-- 2 root 2001 982M Jan 2 15:14 test.img至此，所有的操作都已完成。
免責聲明：本站發布的內容（圖片、視頻和文字）以原創、轉載和分享為主，文章觀點不代表本網站立場，如果涉及侵權請聯系站長郵箱：is@yisu.com進行舉報，并提供相關證據，一經查實，將立刻刪除涉嫌侵權內容。
