使用cephadm部署ceph集群

yum -y install python3

# 安装阿里云提供的docker-ceyum install -y yum-utils device-mapper-persistent-data lvm2yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.reposed -i 's+download.docker.com+mirrors.aliyun.com/docker-ce+' /etc/yum.repos.d/docker-ce.repoyum -y install docker-cesystemctl enable docker --now# 配置镜像加速器mkdir -p /etc/dockertee /etc/docker/daemon.json <<-'EOF'{  "registry-mirrors": ["https://bp1bh1ga.mirror.aliyuncs.com"]}EOFsystemctl daemon-reloadsystemctl restart docker

172.24.1.6 node1172.24.1.7 node2172.24.1.8 node3172.24.1.9 node4

[root@node1 ~]# ssh-keygen[root@node1 ~]# ssh-copy-id root@node2[root@node1 ~]# ssh-copy-id root@node3[root@node1 ~]# ssh-copy-id root@node4

1.安装epel源[root@node1 ~]# yum -y install epel-release2.安装ceph源[root@node1 ~]# yum search release-ceph上次元数据过期检查：0:57:14 前，执行于 2023年02月14日 星期二 14时22分00秒。================= 名称 匹配：release-ceph ============================================centos-release-ceph-nautilus.noarch : Ceph Nautilus packages from the CentOS Storage SIG repositorycentos-release-ceph-octopus.noarch : Ceph Octopus packages from the CentOS Storage SIG repositorycentos-release-ceph-pacific.noarch : Ceph Pacific packages from the CentOS Storage SIG repositorycentos-release-ceph-quincy.noarch : Ceph Quincy packages from the CentOS Storage SIG repository[root@node1 ~]# yum -y install centos-release-ceph-pacific.noarch3.安装cephadm[root@node1 ~]# yum -y install cephadm4.安装ceph-common[root@node1 ~]# yum -y install ceph-common

[root@node1 ~]# cephadm bootstrap --mon-ip 172.24.1.6 --allow-fqdn-hostname --initial-dashboard-user admin --initial-dashboard-password redhat --dashboard-password-noupdateVerifying podman|docker is present...Verifying lvm2 is present...Verifying time synchronization is in place...Unit chronyd.service is enabled and runningRepeating the final host check...docker (/usr/bin/docker) is presentsystemctl is presentlvcreate is presentUnit chronyd.service is enabled and runningHost looks OKCluster fsid: 0b565668-ace4-11ed-960c-5254000de7a0Verifying IP 172.24.1.6 port 3300 ...Verifying IP 172.24.1.6 port 6789 ...Mon IP `172.24.1.6` is in CIDR network `172.24.1.0/24`- internal network (--cluster-network) has not been provided, OSD replication will default to the public_networkPulling container image quay.io/ceph/ceph:v16...Ceph version: ceph version 16.2.11 (3cf40e2dca667f68c6ce3ff5cd94f01e711af894) pacific (stable)Extracting ceph user uid/gid from container image...Creating initial keys...Creating initial monmap...Creating mon...Waiting for mon to start...Waiting for mon...mon is availableAssimilating anything we can from ceph.conf...Generating new minimal ceph.conf...Restarting the monitor...Setting mon public_network to 172.24.1.0/24Wrote config to /etc/ceph/ceph.confWrote keyring to /etc/ceph/ceph.client.admin.keyringCreating mgr...Verifying port 9283 ...Waiting for mgr to start...Waiting for mgr...mgr not available, waiting (1/15)...mgr not available, waiting (2/15)...mgr not available, waiting (3/15)...mgr is availableEnabling cephadm module...Waiting for the mgr to restart...Waiting for mgr epoch 5...mgr epoch 5 is availableSetting orchestrator backend to cephadm...Generating ssh key...Wrote public SSH key to /etc/ceph/ceph.pubAdding key to root@localhost authorized_keys...Adding host node1...Deploying mon service with default placement...Deploying mgr service with default placement...Deploying crash service with default placement...Deploying prometheus service with default placement...Deploying grafana service with default placement...Deploying node-exporter service with default placement...Deploying alertmanager service with default placement...Enabling the dashboard module...Waiting for the mgr to restart...Waiting for mgr epoch 9...mgr epoch 9 is availableGenerating a dashboard self-signed certificate...Creating initial admin user...Fetching dashboard port number...Ceph Dashboard is now available at:
             URL: https://node1.domain1.example.com:8443/            User: admin        Password: redhat
Enabling client.admin keyring and conf on hosts with "admin" labelYou can access the Ceph CLI with:
        sudo /usr/sbin/cephadm shell --fsid 0b565668-ace4-11ed-960c-5254000de7a0 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring
Please consider enabling telemetry to help improve Ceph:
        ceph telemetry on
For more information see:
        https://docs.ceph.com/docs/pacific/mgr/telemetry/
Bootstrap complete.

[root@node1 ~]# ssh-copy-id -f -i /etc/ceph/ceph.pub root@node2[root@node1 ~]# ssh-copy-id -f -i /etc/ceph/ceph.pub root@node3[root@node1 ~]# ssh-copy-id -f -i /etc/ceph/ceph.pub root@node4

[root@node1 ~]# ceph orch host add node2 172.24.1.7Added host 'node2' with addr '172.24.1.7'[root@node1 ~]# ceph orch host add node3 172.24.1.8Added host 'node3' with addr '172.24.1.8'[root@node1 ~]# ceph orch host add node4 172.24.1.9Added host 'node4' with addr '172.24.1.9'

[root@node1 ~]# ceph orch host label add node1 _adminAdded label _admin to host node1[root@node1 ~]# ceph orch host label add node4 _adminAdded label _admin to host node4[root@node1 ~]# scp /etc/ceph/{*.conf,*.keyring} root@node4:/etc/ceph[root@node1 ~]# ceph orch host lsHOST   ADDR        LABELS  STATUS  node1  172.24.1.6  _admin          node2  172.24.1.7                  node3  172.24.1.8                  node4  172.24.1.9  _admin

[root@node1 ~]# ceph orch apply mon "node1,node2,node3"Scheduled mon update...

[root@node1 ~]# ceph orch apply mgr --placement="node1,node2,node3"Scheduled mgr update...

[root@node1 ~]# ceph orch daemon add osd node1:/dev/vdb[root@node1 ~]# ceph orch daemon add osd node1:/dev/vdc[root@node1 ~]# ceph orch daemon add osd node1:/dev/vdd[root@node1 ~]# ceph orch daemon add osd node2:/dev/vdb[root@node1 ~]# ceph orch daemon add osd node2:/dev/vdc[root@node1 ~]# ceph orch daemon add osd node2:/dev/vdd[root@node1 ~]# ceph orch daemon add osd node3:/dev/vdb[root@node1 ~]# ceph orch daemon add osd node3:/dev/vdc[root@node1 ~]# ceph orch daemon add osd node3:/dev/vdd或者：[root@node1 ~]# for i in node1 node2 node3; do for j in vdb vdc vdd; do ceph orch daemon add osd $i:/dev/$j; done; doneCreated osd(s) 0 on host 'node1'Created osd(s) 1 on host 'node1'Created osd(s) 2 on host 'node1'Created osd(s) 3 on host 'node2'Created osd(s) 4 on host 'node2'Created osd(s) 5 on host 'node2'Created osd(s) 6 on host 'node3'Created osd(s) 7 on host 'node3'Created osd(s) 8 on host 'node3'
[root@node1 ~]# ceph orch device lsHOST   PATH      TYPE  DEVICE ID   SIZE  AVAILABLE  REFRESHED  REJECT REASONS                                                 node1  /dev/vdb  hdd              10.7G             4m ago     Insufficient space (<10 extents) on vgs, LVM detected, locked  node1  /dev/vdc  hdd              10.7G             4m ago     Insufficient space (<10 extents) on vgs, LVM detected, locked  node1  /dev/vdd  hdd              10.7G             4m ago     Insufficient space (<10 extents) on vgs, LVM detected, locked  node2  /dev/vdb  hdd              10.7G             3m ago     Insufficient space (<10 extents) on vgs, LVM detected, locked  node2  /dev/vdc  hdd              10.7G             3m ago     Insufficient space (<10 extents) on vgs, LVM detected, locked  node2  /dev/vdd  hdd              10.7G             3m ago     Insufficient space (<10 extents) on vgs, LVM detected, locked  node3  /dev/vdb  hdd              10.7G             90s ago    Insufficient space (<10 extents) on vgs, LVM detected, locked  node3  /dev/vdc  hdd              10.7G             90s ago    Insufficient space (<10 extents) on vgs, LVM detected, locked  node3  /dev/vdd  hdd              10.7G             90s ago    Insufficient space (<10 extents) on vgs, LVM detected, locked

[root@node1 ~]# ceph -s  cluster:    id:     0b565668-ace4-11ed-960c-5254000de7a0    health: HEALTH_OK   services:    mon: 3 daemons, quorum node1,node2,node3 (age 7m)    mgr: node1.cxtokn(active, since 14m), standbys: node2.heebcb, node3.fsrlxu    osd: 9 osds: 9 up (since 59s), 9 in (since 81s)   data:    pools:   1 pools, 1 pgs    objects: 0 objects, 0 B    usage:   53 MiB used, 90 GiB / 90 GiB avail    pgs:     1 active+clean

# 在目录5.4已经将ceph配置文件和keyring拷贝到node4节点[root@node4 ~]# ceph -s-bash: ceph: 未找到命令，需要安装ceph-common# 安装ceph源[root@node4 ~]# yum -y install centos-release-ceph-pacific.noarch# 安装ceph-common[root@node4 ~]# yum -y install ceph-common[root@node4 ~]# ceph -s  cluster:    id:     0b565668-ace4-11ed-960c-5254000de7a0    health: HEALTH_OK   services:    mon: 3 daemons, quorum node1,node2,node3 (age 7m)    mgr: node1.cxtokn(active, since 14m), standbys: node2.heebcb, node3.fsrlxu    osd: 9 osds: 9 up (since 59s), 9 in (since 81s)   data:    pools:   1 pools, 1 pgs    objects: 0 objects, 0 B    usage:   53 MiB used, 90 GiB / 90 GiB avail    pgs:     1 active+clean