[root@bs-k8s-ceph ~]# ceph -s cluster: id: 11880418-1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: 3 daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-hk-hk02(active), standbys: bs-k8s-ceph, bs-hk-hk01 osd: 6 osds: 6 up, 6 in data: pools: 3 pools, 320 pgs objects: 416 objects, 978 MiB usage: 8.7 GiB used, 105 GiB / 114 GiB avail pgs: 320 active+clean[root@bs-k8s-master01 ~]# kubectl get pods --all-namespaces NAMESPACE NAME READY STATUS RESTARTS AGEdefault rbd-provisioner-75b85f85bd-8ftdm 0/1 Terminating 12 5ddefault rbd-provisioner-75b85f85bd-bk5kz 1/1 Running 4 24hharbor rbd-provisioner-75b85f85bd-dhnr4 1/1 Terminating 7 3d13hharbor rbd-provisioner-75b85f85bd-qzzbx 1/1 Running 3 24hkube-system calico-kube-controllers-5b644bc49c-kv4wb 1/1 Running 67 10dkube-system calico-node-4jxbp 1/1 Running 5 10dkube-system calico-node-7t9cj 1/1 Running 12 10dkube-system calico-node-cchgl 1/1 Running 15 10dkube-system calico-node-czj76 1/1 Running 7 10dkube-system calico-node-lxb2s 0/1 Running 15 10dkube-system calico-node-nmg9t 1/1 Running 8 10dkube-system coredns-7f9c544f75-bwx9p 1/1 Running 67 10dkube-system coredns-7f9c544f75-q58mr 1/1 Running 66 10dkube-system dashboard-metrics-scraper-6b66849c9-qtwzx 1/1 Running 83 10dkube-system etcd-bs-k8s-master01 1/1 Running 30 10dkube-system etcd-bs-k8s-master02 1/1 Running 12 10dkube-system etcd-bs-k8s-master03 1/1 Running 37 10dkube-system kube-apiserver-bs-k8s-master01 1/1 Running 49 10dkube-system kube-apiserver-bs-k8s-master02 1/1 Running 22 10dkube-system kube-apiserver-bs-k8s-master03 1/1 Running 67 10dkube-system kube-controller-manager-bs-k8s-master01 1/1 Running 49 10dkube-system kube-controller-manager-bs-k8s-master02 1/1 Running 38 10dkube-system kube-controller-manager-bs-k8s-master03 1/1 Running 42 10dkube-system kube-proxy-26ffm 1/1 Running 4 10dkube-system kube-proxy-298tr 1/1 Running 5 10dkube-system kube-proxy-hzsmb 1/1 Running 4 10dkube-system kube-proxy-jb4sq 1/1 Running 5 10dkube-system kube-proxy-pt94r 1/1 Running 7 10dkube-system kube-proxy-wljwv 1/1 Running 5 10dkube-system kube-scheduler-bs-k8s-master01 1/1 Running 46 10dkube-system kube-scheduler-bs-k8s-master02 1/1 Running 31 10dkube-system kube-scheduler-bs-k8s-master03 1/1 Running 46 10dkube-system kubernetes-dashboard-887cbd9c6-j7ptq 1/1 Terminating 30 10dkube-system kubernetes-dashboard-887cbd9c6-rzlp4 1/1 Running 7 24hkube-system rbd-provisioner-75b85f85bd-2dclz 1/1 Running 35 3d14h[root@bs-k8s-ceph ~]# ceph osd pool create gitlab 64pool ‘github‘ created[root@bs-k8s-ceph ~]# cd /etc/ceph/[root@bs-k8s-ceph ceph]# ceph auth get-or-create client.gitlab mon ‘allow r‘ osd ‘allow class-read, allow rwx pool=gitlab‘ -o ceph.client.gitlab.keyring[root@bs-k8s-node01 ~]# ceph auth get-key client.admin | base64QVFDNmNVSmV2eU8yRnhBQVBxYzE5Mm5PelNnZk5acmg5aEFQYXc9PQ==[root@bs-k8s-node01 ~]# ceph auth get-key client.gitlab | base64QVFCUzNFMWVqRDZpTXhBQTgrd3RIajVWb295M1J1QkJDZi9QOXc9PQ==[root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-gitlab-namespace.yamlnamespace/gitlab created[root@bs-k8s-master01 gitlab]# kubectl get namespace gitlabNAME STATUS AGEgitlab Active <invalid>[root@bs-k8s-master01 gitlab]# cat ceph-gitlab-namespace.yaml ###########################################################################Author: zisefeizhu#QQ: 2********0#Date: 2020-02-16#FileName: ceph-harbor-namespace.yaml#URL: https://www.cnblogs.com/zisefeizhu/#Description: The test script#Copyright (C): 2020 All rights reserved###########################################################################apiVersion: v1kind: Namespacemetadata: name: gitlab[root@bs-k8s-node02 ~]# docker tag quay.io/external_storage/rbd-provisioner:latest harbor.linux.com/kubworker/rbd-provisioner:latest[root@bs-k8s-node02 ~]# docker push harbor.linux.com/kubworker/rbd-provisioner:latest[root@bs-k8s-master01 gitlab]# kubectl apply -f external-storage-rbd-provisioner.yamlserviceaccount/rbd-provisioner createdclusterrole.rbac.authorization.k8s.io/rbd-provisioner createdclusterrolebinding.rbac.authorization.k8s.io/rbd-provisioner createdrole.rbac.authorization.k8s.io/rbd-provisioner createdrolebinding.rbac.authorization.k8s.io/rbd-provisioner createddeployment.apps/rbd-provisioner created[root@bs-k8s-master01 gitlab]# kubectl get pods -n gitlab NAME READY STATUS RESTARTS AGErbd-provisioner-75b85f85bd-t224w 1/1 Running 0 64s[root@bs-k8s-master01 gitlab]# cat external-storage-rbd-provisioner.yamlapiVersion: v1kind: ServiceAccountmetadata: name: rbd-provisioner namespace: gitlab---kind: ClusterRoleapiVersion: rbac.authorization.k8s.io/v1metadata: name: rbd-provisionerrules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["create", "update", "patch"] - apiGroups: [""] resources: ["endpoints"] verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["services"] resourceNames: ["kube-dns"] verbs: ["list", "get"]---kind: ClusterRoleBindingapiVersion: rbac.authorization.k8s.io/v1metadata: name: rbd-provisionersubjects: - kind: ServiceAccount name: rbd-provisioner namespace: gitlabroleRef: kind: ClusterRole name: rbd-provisioner apiGroup: rbac.authorization.k8s.io---apiVersion: rbac.authorization.k8s.io/v1kind: Rolemetadata: name: rbd-provisioner namespace: gitlabrules:- apiGroups: [""] resources: ["secrets"] verbs: ["get"]---apiVersion: rbac.authorization.k8s.io/v1kind: RoleBindingmetadata: name: rbd-provisioner namespace: gitlabroleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: rbd-provisionersubjects:- kind: ServiceAccount name: rbd-provisioner namespace: gitlab---apiVersion: apps/v1kind: Deploymentmetadata: name: rbd-provisioner namespace: gitlabspec: replicas: 1 selector: matchLabels: app: rbd-provisioner strategy: type: Recreate template: metadata: labels: app: rbd-provisioner spec: containers: - name: rbd-provisioner image: "harbor.linux.com/kubworker/rbd-provisioner:latest" env: - name: PROVISIONER_NAME value: ceph.com/rbd serviceAccount: rbd-provisioner imagePullSecrets: - name: login[root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-gitlab-secret.yamlsecret/ceph-gitlab-admin-secret createdsecret/ceph-gitlab-gitlab-secret created[root@bs-k8s-master01 gitlab]# cat ceph-gitlab-secret.yaml ###########################################################################Author: zisefeizhu#QQ: 2********0#Date: 2020-02-16#FileName: ceph-harbor-secret.yaml#URL: https://www.cnblogs.com/zisefeizhu/#Description: The test script#Copyright (C): 2020 All rights reserved###########################################################################apiVersion: v1kind: Secretmetadata: name: ceph-gitlab-admin-secret namespace: gitlabdata: key: QVFDNmNVSmV2eU8yRnhBQVBxYzE5Mm5PelNnZk5acmg5aEFQYXc9PQ==type: kubernetes.io/rbd---apiVersion: v1kind: Secretmetadata: name: ceph-gitlab-gitlab-secret namespace: gitlabdata: key: QVFCUzNFMWVqRDZpTXhBQTgrd3RIajVWb295M1J1QkJDZi9QOXc9PQ==type: kubernetes.io/rbd[root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-gitlab-storageclass.yaml storageclass.storage.k8s.io/ceph-gitlab created[root@bs-k8s-master01 gitlab]# cat ceph-gitlab-storageclass.yaml ###########################################################################Author: zisefeizhu#QQ: 2********0#Date: 2020-02-16#FileName: ceph-harbor-storageclass.yaml#URL: https://www.cnblogs.com/zisefeizhu/#Description: The test script#Copyright (C): 2020 All rights reserved###########################################################################apiVersion: storage.k8s.io/v1kind: StorageClassmetadata: name: ceph-gitlab annotations: storageclass.kubernetes.io/is-default-class: "false"provisioner: ceph.com/rbdreclaimPolicy: Retainparameters: monitors: 20.0.0.206:6789,20.0.0.207:6789,20.0.0.208:6789 adminId: admin adminSecretName: ceph-gitlab-admin-secret adminSecretNamespace: gitlab pool: gitlab fsType: xfs userId: gitlab userSecretName: ceph-gitlab-gitlab-secret imageFormat: "2" imageFeatures: "layering"[root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-gitlab-pvc.yaml persistentvolumeclaim/pvc-ceph-gitlab created[root@bs-k8s-master01 gitlab]# kubectl get pv -n gitlabNAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGEpvc-494a130d-018c-4be3-9b31-e951cc4367a5 20Gi RWO Retain Bound default/wp-pv-claim ceph-rbd 4d14hpvc-8ffa3182-a2f6-47d9-a71d-ff8e8b379a16 1Gi RWO Retain Bound default/ceph-pvc ceph-rbd 4d16hpvc-ac7d3a09-123e-4614-886c-cded8822a078 20Gi RWO Retain Bound default/mysql-pv-claim ceph-rbd 4d14hpvc-ba59cba5-e394-4b19-8c4b-4018162bbe69 1Gi RWO Retain Bound harbor/pvc-ceph-harbor ceph-harbor 3d14h[root@bs-k8s-master01 gitlab]# kubectl get pvc -n gitlabNAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGEpvc-ceph-gitlab Bound pvc-c7c7aed3-4f8f-4801-a9db-c13f9390af8c 1Gi RWO ceph-gitlab 21s[root@bs-k8s-master01 gitlab]# cat ceph-gitlab-pvc.yaml ###########################################################################Author: zisefeizhu#QQ: 2********0#Date: 2020-02-16#FileName: ceph-harbor-pvc.yaml#URL: https://www.cnblogs.com/zisefeizhu/#Description: The test script#Copyright (C): 2020 All rights reserved###########################################################################apiVersion: v1kind: PersistentVolumeClaimmetadata: name: pvc-ceph-gitlab namespace: gitlabspec: storageClassName: ceph-gitlab accessModes: - ReadWriteOnce resources: requests: storage: 1Gi[root@bs-k8s-ceph ceph]# ceph osd pool application enable gitlab rbdenabled application ‘rbd‘ on pool ‘gitlab‘[root@bs-k8s-ceph ceph]# ceph -s cluster: id: 11880418-1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: 3 daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-hk-hk02(active), standbys: bs-k8s-ceph, bs-hk-hk01 osd: 6 osds: 6 up, 6 in data: pools: 4 pools, 384 pgs objects: 420 objects, 978 MiB usage: 8.9 GiB used, 105 GiB / 114 GiB avail pgs: 384 active+clean[root@bs-k8s-master01 gitlab]# echo -n "admin" > ./username[root@bs-k8s-master01 gitlab]# echo -n "zisefeizhzu" > ./password[root@bs-k8s-master01 gitlab]# kubectl create secret generic db-user-pass --from-file=./username --from-file=./passwordsecret/db-user-pass created[root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-redis-pvc.yamlpersistentvolumeclaim/pvc-ceph-redis created[root@bs-k8s-master01 gitlab]# kubectl get pvc -n gitlabNAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGEpvc-ceph-gitlab Bound pvc-c7c7aed3-4f8f-4801-a9db-c13f9390af8c 1Gi RWO ceph-gitlab 53mpvc-ceph-redis Bound pvc-f8051bd4-4617-4823-b190-3581ec502d7e 1Gi RWO ceph-gitlab 15s[root@bs-k8s-master01 gitlab]# cat ceph-redis-pvc.yaml ###########################################################################Author: zisefeizhu#QQ: 2********0#Date: 2020-02-20#FileName: ceph-redis-pvc.yaml#URL: https://www.cnblogs.com/zisefeizhu/#Description: The test script#Copyright (C): 2020 All rights reserved###########################################################################apiVersion: v1kind: PersistentVolumeClaimmetadata: name: pvc-ceph-redis namespace: gitlabspec: storageClassName: ceph-gitlab accessModes: - ReadWriteOnce resources: requests: storage: 1Gi[root@bs-k8s-master01 gitlab]# kubectl apply -f gitlab-redis.yamldeployment.apps/redis createdservice/redis created[root@bs-k8s-master01 gitlab]# kubectl get pods -n gitlab -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATESrbd-provisioner-75b85f85bd-t224w 1/1 Running 0 81m 10.209.145.42 bs-k8s-node02 <none> <none>redis-598f9d7ccb-vmrpz 1/1 Running 0 2m7s 10.209.208.22 bs-k8s-node03 <none> <none>[root@bs-k8s-master01 gitlab]# cat gitlab-redis.yaml ###########################################################################Author: zisefeizhu#QQ: 2********0#Date: 2020-02-20#FileName: gitlab-redis.yaml#URL: https://www.cnblogs.com/zisefeizhu/#Description: The test script#Copyright (C): 2020 All rights reserved###########################################################################apiVersion: apps/v1kind: Deploymentmetadata: name: redis namespace: gitlab labels: name: redisspec: selector: matchLabels: name: redis template: metadata: name: redis labels: name: redis spec: containers: - name: redis image: sameersbn/redis imagePullPolicy: IfNotPresent ports: - name: redis containerPort: 6379 volumeMounts: - mountPath: /var/lib/redis name: data livenessProbe: exec: command: - redis-cli - ping initialDelaySeconds: 30 timeoutSeconds: 5 readinessProbe: exec: command: - redis-cli - ping initialDelaySeconds: 5 timeoutSeconds: 1 volumes: - name: data persistentVolumeClaim: claimName: pvc-ceph-redis---apiVersion: v1kind: Servicemetadata: name: redis namespace: gitlab labels: name: redisspec: ports: - name: redis port: 6379 targetPort: redis selector: name: redis[root@bs-k8s-master01 gitlab]# cp ceph-redis-pvc.yaml ceph-postgresql-pvc.yaml[root@bs-k8s-master01 gitlab]# vim ceph-postgresql-pvc.yaml[root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-postgresql-pvc.yamlpersistentvolumeclaim/pvc-ceph-postgresql created[root@bs-k8s-master01 gitlab]# kubectl get pvc -n gitlabNAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGEpvc-ceph-gitlab Bound pvc-c7c7aed3-4f8f-4801-a9db-c13f9390af8c 1Gi RWO ceph-gitlab 23hpvc-ceph-postgresql Bound pvc-5995ff51-08fe-4f66-aca8-bda906c44c83 1Gi RWO ceph-gitlab 21hpvc-ceph-redis Bound pvc-f8051bd4-4617-4823-b190-3581ec502d7e 1Gi RWO ceph-gitlab 22h[root@bs-k8s-master01 gitlab]# cat ceph-postgresql-pvc.yaml ###########################################################################Author: zisefeizhu#QQ: 2********0#Date: 2020-02-20#FileName: ceph-redis-pvc.yaml#URL: https://www.cnblogs.com/zisefeizhu/#Description: The test script#Copyright (C): 2020 All rights reserved###########################################################################apiVersion: v1kind: PersistentVolumeClaimmetadata: name: pvc-ceph-postgresql namespace: gitlabspec: storageClassName: ceph-gitlab accessModes: - ReadWriteOnce resources: requests: storage: 1Gi[root@bs-k8s-master01 gitlab]# kubectl apply -f gitlab-postgresql.yamldeployment.apps/postgresql createdservice/postgresql created[root@bs-k8s-master01 gitlab]# kubectl get pods -n gitlabNAME READY STATUS RESTARTS AGEpostgresql-8c84b7698-w4wpp 1/1 Running 0 5m31srbd-provisioner-75b85f85bd-t224w 1/1 Running 0 104mredis-598f9d7ccb-vmrpz 1/1 Running 0 24m[root@bs-k8s-master01 gitlab]# cat gitlab-postgresql.yaml ###########################################################################Author: zisefeizhu#QQ: 2********0#Date: 2020-02-20#FileName: gitlab-postgresql.yaml#URL: https://www.cnblogs.com/zisefeizhu/#Description: The test script#Copyright (C): 2020 All rights reserved###########################################################################apiVersion: apps/v1kind: Deploymentmetadata: name: postgresql namespace: gitlab labels: name: postgresqlspec: selector: matchLabels: name: postgresql template: metadata: name: postgresql labels: name: postgresql spec: containers: - name: postgresql image: sameersbn/postgresql imagePullPolicy: IfNotPresent env: - name: DB_USER value: gitlab - name: DB_PASS value: zisefeizhu - name: DB_NAME value: gitlab_production - name: DB_EXTENSION value: pg_trgm ports: - name: postgres containerPort: 5432 volumeMounts: - mountPath: /var/lib/postgresql name: data livenessProbe: exec: command: - pg_isready - -h - localhost - -U - postgres initialDelaySeconds: 30 timeoutSeconds: 5 readinessProbe: exec: command: - pg_isready - -h - localhost - -U - postgres initialDelaySeconds: 5 timeoutSeconds: 1 volumes: - name: data persistentVolumeClaim: claimName: pvc-ceph-postgresql---apiVersion: v1kind: Servicemetadata: name: postgresql namespace: gitlab labels: name: postgresqlspec: ports: - name: postgres port: 5432 targetPort: postgres selector: name: postgresql[root@bs-k8s-master01 gitlab]# kubectl apply -f gitlab-gitlab.yamldeployment.apps/gitlab createdservice/gitlab createdingress.extensions/gitlab created[root@bs-k8s-master01 gitlab]# kubectl get ingress -n gitlabNAME HOSTS ADDRESS PORTS AGEgitlab gitlab.linux.com 80 106s[root@bs-k8s-master01 gitlab]# kubectl get service -n gitlabNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGEgitlab NodePort 10.96.120.85 <none> 80:32208/TCP,22:30022/TCP 2m33spostgresql ClusterIP 10.111.30.80 <none> 5432/TCP 41mredis ClusterIP 10.103.127.83 <none> 6379/TCP [root@bs-k8s-master01 gitlab]# kubectl get pods -n gitlab -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATESgitlab-6468b9f55c-mzcmj 1/1 Running 1 4m59s 10.209.208.41 bs-k8s-node03 <none> <none>postgresql-8c84b7698-w4wpp 1/1 Running 0 101m 10.209.145.43 bs-k8s-node02 <none> <none>rbd-provisioner-75b85f85bd-t224w 1/1 Running 0 3h20m 10.209.145.42 bs-k8s-node02 <none> <none>redis-598f9d7ccb-vmrpz 1/1 Running 0 120m 10.209.208.22 bs-k8s-node03 <none> [root@bs-k8s-master01 gitlab]# cat gitlab-gitlab.yaml ###########################################################################Author: zisefeizhu#QQ: 2********0#Date: 2020-02-20#FileName: gitlab-gitlab.yaml#URL: https://www.cnblogs.com/zisefeizhu/#Description: The test script#Copyright (C): 2020 All rights reserved###########################################################################apiVersion: apps/v1kind: Deploymentmetadata: name: gitlab namespace: gitlab labels: name: gitlabspec: selector: matchLabels: name: gitlab template: metadata: name: gitlab labels: name: gitlab spec: containers: - name: gitlab image: sameersbn/gitlab:12.1.6 imagePullPolicy: IfNotPresent env: - name: TZ value: Asia/Shanghai - name: GITLAB_TIMEZONE value: Beijing - name: GITLAB_SECRETS_DB_KEY_BASE value: long-and-random-alpha-numeric-string - name: GITLAB_SECRETS_SECRET_KEY_BASE value: long-and-random-alpha-numeric-string - name: GITLAB_SECRETS_OTP_KEY_BASE value: long-and-random-alpha-numeric-string - name: GITLAB_ROOT_PASSWORD value: zisefeizhu - name: GITLAB_ROOT_EMAIL value: 2350835860@qq.com - name: GITLAB_HOST value: gitlab.linux.com - name: GITLAB_PORT value: "80" - name: GITLAB_SSH_PORT value: "30022" - name: GITLAB_NOTIFY_ON_BROKEN_BUILDS value: "true" - name: GITLAB_NOTIFY_PUSHER value: "false" - name: GITLAB_BACKUP_SCHEDULE value: daily - name: GITLAB_BACKUP_TIME value: 01:00 - name: DB_TYPE value: postgres - name: DB_HOST value: postgresql - name: DB_PORT value: "5432" - name: DB_USER value: gitlab - name: DB_PASS value: zisefeizhu - name: DB_NAME value: gitlab_production - name: REDIS_HOST value: redis - name: REDIS_PORT value: "6379" ports: - name: http containerPort: 80 - name: ssh containerPort: 22 volumeMounts: - mountPath: /home/git/data name: data livenessProbe: httpGet: path: / port: 80 initialDelaySeconds: 180 timeoutSeconds: 5 readinessProbe: httpGet: path: / port: 80 initialDelaySeconds: 5 timeoutSeconds: 1 volumes: - name: data persistentVolumeClaim: claimName: pvc-ceph-gitlab---apiVersion: v1kind: Servicemetadata: name: gitlab namespace: gitlab labels: name: gitlabspec: ports: - name: http port: 80 targetPort: http - name: ssh port: 22 targetPort: ssh nodePort: 30022 type: NodePort selector: name: gitlab---apiVersion: extensions/v1beta1kind: Ingressmetadata: name: gitlab namespace: gitlab annotations: kubernetes.io/ingress.class: traefikspec: rules: - host: gitlab.linux.com http: paths: - backend: serviceName: gitlab servicePort: http