1.准备环境 关闭selinux、防火墙、配置时间同步、新建cephadm

cephadm执行:
yum  install -y ansible
vim /etc/ansible/ansible.cfg
module_set_locale = False
vim /etc/ansible/hosts
[cephadm]
10.0.0.51 host=cephadm.local.cn ansible_connection=local
[ceph]
10.0.0.52 host=mon01
10.0.0.53 host=mon02
10.0.0.54 host=mon03
10.0.0.55 host=stor04
[ceph:vars]
domain=local.cn
#新建普通用户安装ceph 赋予sudo root权限无密码
useradd cephadm && echo "456" |passwd --stdin cephadm
vim /etc/sudoers.d/cephadm 
cephadm  ALL=(root)  NOPASSWD:  ALL
vim /etc/hosts
10.0.0.51 cephadm
10.0.0.52 mon01
10.0.0.51 cephadm
10.0.0.52 mon01
10.0.0.53 mon02
10.0.0.54 mon03
10.0.0.55 stor04

ansible all -k  -m lineinfile -a "path=/etc/selinux/config regexp='^SELINUX=' line='SELINUX=disabled'"
ansible all -k -m shell -a "setenforce 0"
ansible all -k -m service -a "name=firewalld state=stopped enabled=no"
ansible all -k -m shell -a "useradd cephadm && echo "456" |passwd --stdin cephadm"
ansible all -k  -m copy -a "src=/etc/sudoers.d/cephadm dest=/etc/sudoers.d/cephadm"
ansible all -k  -m copy -a "src=/etc/hosts dest=/etc/hosts"
ansible all -k -m hostname -a 'name={{ host }}.{{ domain }}'
ansible 10.0.0.51 -m replace -a "path=/etc/chrony.conf regexp='(server 0.*)' replace='server ntp.aliyun.com iburst'"
ansible 10.0.0.51 -m replace -a "path=/etc/chrony.conf regexp='(server [1-3].*)' replace='#\1'"
ansible 10.0.0.51 -m replace -a "path=/etc/chrony.conf regexp='(#allow.*)' replace='allow 10.0.0.0/24'"
ansible ceph -k -m replace -a "path=/etc/chrony.conf regexp='(server 0.*)' replace='server 10.0.0.51 iburst'"
ansible ceph -k -m replace -a "path=/etc/chrony.conf regexp='(server [1-3].*)' replace='#\1'"
ansible -k all -m service -a "name=chronyd state=restarted"
#配置各个主机之间cephadm ssh免密登录 
su - cephadm
cat  > ~/.ssh/config <<EOF
StrictHostKeyChecking no
EOF
chmod 400 ~/.ssh/config
vim sshkey.sh
#!/bin/bash
[ ! -f ~/.ssh/id_rsa ] && ssh-keygen -f ~/.ssh/id_rsa -P ''
NET=10.0.0
export SSHPASS=456
for IP in {51..55};do
  sshpass -e ssh-copy-id $NET.$IP
done

2.部署ceph mimic

2.1 初始化ceph

cephadm:
rpm -ivh http://mirrors.aliyun.com/ceph/rpm-mimic/el7/noarch/ceph-release-1-1.el7.noarch.rpm
yum install -y epel-release
 yum install ceph-deploy python-setuptools python2-subprocess32 -y

[root@cephadm ~]# su - cephadm 
[cephadm@cephadm ~]$ mkdir ceph-cluster
[cephadm@cephadm ~]$ cd ceph-cluster
ceph-deploy new --help  查看帮助
[cephadm@cephadm ceph-cluster]$ ceph-deploy new --cluster-network 192.168.199.0/24 --public-network=10.0.0.0/24 mon01  ##初始化mon节点
[cephadm@cephadm ceph-cluster]$ ll
-rw-rw-r-- 1 cephadm cephadm  256 3月  31 15:36 ceph.conf
-rw-rw-r-- 1 cephadm cephadm 3264 3月  31 15:36 ceph-deploy-ceph.log
-rw------- 1 cephadm cephadm   73 3月  31 15:36 ceph.mon.keyring
[cephadm@cephadm ceph-cluster]$ cat ceph.conf   开始未选择网络 编辑此配置文件也可以
[global]
fsid = f59d07d8-b127-46f7-b990-f34083581cfc
public_network = 10.0.0.0/24
cluster_network = 192.168.199.0/24
mon_initial_members = mon01
mon_host = 10.0.0.52
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

2.2 安装ceph集群 在mon1 mon2 mon3 stor4

[cephadm@cephadm ceph-cluster]$ ceph-deploy install mon01 mon02 mon3 stor4
相当于 ssh远程主机执行
rpm -ivh http://mirrors.aliyun.com/ceph/rpm-mimic/el7/noarch/ceph-release-1-1.el7.noarch.rpm
yum -y install epel-release
yum install -y ceph ceph-radosgw
ceph-deploy install --no-adjust-repos mon01 mon02 mon3 stor4 

2.3 配置初始化mon节点 并收集所有密钥

[cephadm@cephadm ceph-cluster]$ ceph-deploy mon create-initial #默认读取配置文件完成
[cephadm@cephadm ceph-cluster]$ ll
总用量 220
-rw------- 1 cephadm cephadm    113 3月  31 16:36 ceph.bootstrap-mds.keyring
-rw------- 1 cephadm cephadm    113 3月  31 16:36 ceph.bootstrap-mgr.keyring
-rw------- 1 cephadm cephadm    113 3月  31 16:36 ceph.bootstrap-osd.keyring
-rw------- 1 cephadm cephadm    113 3月  31 16:36 ceph.bootstrap-rgw.keyring
-rw------- 1 cephadm cephadm    151 3月  31 16:36 ceph.client.admin.keyring
-rw-rw-r-- 1 cephadm cephadm    256 3月  31 15:36 ceph.conf
-rw-rw-r-- 1 cephadm cephadm 124475 3月  31 16:36 ceph-deploy-ceph.log
-rw------- 1 cephadm cephadm     73 3月  31 15:36 ceph.mon.keyring

2.4 将配置文件和admin秘钥拷贝到其他节点

如果不做交互式可以只复制config
[cephadm@cephadm ceph-cluster]$ ceph-deploy admin mon01 mon02 mon03 stor04
[root@mon01 ~]# ll /etc/ceph/ #cephadm无法访问
总用量 12
-rw------- 1 root root 151 3月  31 16:39 ceph.client.admin.keyring
-rw-r--r-- 1 root root 256 3月  31 16:39 ceph.conf 
-rw-r--r-- 1 root root  92 4月  24 2020 rbdmap
-rw------- 1 root root   0 3月  31 16:36 tmpjtTfFI

setfacl -m u:cephadm:rw /etc/ceph/ceph.client.admin.keyring  授权访问

2.5 配置manager节点（仅luminious以上版本）

[cephadm@cephadm ceph-cluster]$ ceph-deploy mgr create  mon03

2.6 查看ceph状态

[cephadm@cephadm ceph-cluster]$ sudo yum install ceph-common -y
[cephadm@cephadm ceph-cluster]$ ceph-deploy admin cephadm
[cephadm@cephadm ceph-cluster]$ sudo setfacl -m u:cephadm:rw /etc/ceph/ceph.client.admin.keyring
[cephadm@cephadm ceph-cluster]$ ceph -s
  cluster:
    id:     f59d07d8-b127-46f7-b990-f34083581cfc
    health: HEALTH_WARN
            OSD count 0 < osd_pool_default_size 3
 
  services:
    mon: 1 daemons, quorum mon01
    mgr: mon03(active)
    osd: 0 osds: 0 up, 0 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0  objects, 0 B
    usage:   0 B used, 0 B / 0 B avail
    pgs:     

2.7 向RADOS集群添加OSD

擦写磁盘文件数据
ceph-deploy disk zap mon01 /dev/vda
ceph-deploy disk zap mon02 /dev/vda
ceph-deploy disk zap mon03 /dev/vda
ceph-deploy disk zap stor04 /dev/vda
添加磁盘
[cephadm@cephadm ceph-cluster]$ ceph-deploy osd --help
usage: ceph-deploy osd [-h] {list,create} ...
    ceph-deploy osd create {node} --data /path/to/data --block-db /path/to/db-device
    ceph-deploy osd create {node} --data /path/to/data --block-wal /path/to/wal-device
    ceph-deploy osd create {node} --data /path/to/data --block-db /path/to/db-device --block-wal /path/to/wal-device  ##block-db osd源数据 block-wal osd的日志文件
ceph-deploy osd create mon01 --data /dev/vda
ceph-deploy osd create mon02 --data /dev/vda
ceph-deploy osd create mon03 --data /dev/vda
ceph-deploy osd create stor04 --data /dev/vda

[cephadm@cephadm ceph-cluster]$ ceph -s
  cluster:
    id:     f59d07d8-b127-46f7-b990-f34083581cfc
    health: HEALTH_OK
  services:
    mon: 1 daemons, quorum mon01
    mgr: mon03(active)
    osd: 4 osds: 4 up, 4 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0  objects, 0 B
    usage:   4.0 GiB used, 156 GiB / 160 GiB avail
    pgs:     
查看命令
ceph osd stat
ceph osd dump
ceph osd ls

2.8 向ceph添加mon节点冗余 一般奇数

[cephadm@cephadm ceph-cluster]$ ceph-deploy mon add mon02
[cephadm@cephadm ceph-cluster]$ ceph-deploy mon add mon03
[cephadm@cephadm ceph-cluster]$ ceph quorum_status --format json-pretty 以友好显示mon节点

2.9 向ceph添加mgr节点冗余

[cephadm@cephadm ceph-cluster]$ ceph-deploy mgr create stor04
[cephadm@cephadm ceph-cluster]$ ceph -s
  cluster:
    id:     f59d07d8-b127-46f7-b990-f34083581cfc
    health: HEALTH_OK
 
  services:
    mon: 3 daemons, quorum mon01,mon02,mon03
    mgr: mon03(active), standbys: stor04
    osd: 4 osds: 4 up, 4 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0  objects, 0 B
    usage:   4.0 GiB used, 156 GiB / 160 GiB avail
    pgs:     

2.10 移除RADOS集群osd的方法

# 使OSD进入out状态
[root@cephL ceph-deploy]# ceph osd out 0
marked out osd.0.
# 观察数据迁移
[root@cephL ceph-deploy]# ceph -w
# 停止对应的OSD进程
[root@cephL ceph-deploy]# sudo systemctl stop ceph-osd@0
# 清除数据
[root@cephL ceph-deploy]# ceph osd purge 0 --yes-i-really-mean-it
purged osd.0
# 在ceph.conf中移除osd配置
[root@cephL ceph-deploy]# vi /etc/ceph/ceph.conf 

2.11 配置dashboard

https://docs.ceph.com/en/latest/mgr/dashboard/

ceph mgr module enable dashboard  #启用dashboard
ceph config set mgr mgr/dashboard/ssl false #禁用加密
#重启服务
eph mgr module disable dashboard
ceph mgr module enable dashboard
#配置IP和端口
ceph config set mgr mgr/dashboard/mon03/server_addr 10.0.0.54
ceph config set mgr mgr/dashboard/mon03/server_port 8080
ceph config set mgr mgr/dashboard/mon03/ssl_server_port 8443
#配置用户
ceph dashboard set-login-credentials test test
[root@mon03 ~]# ceph mgr services
{
    "dashboard": "http://mon03.local.cn:8080/"
}

图片.png

3. 客户端使用

3.1 挂载rbd块

创建pool 通常在创建pool之前，需要覆盖默认的pg_num，官方推荐：
若少于5个OSD， 设置pg_num为128。
5~10个OSD，设置pg_num为512。
10~50个OSD，设置pg_num为4096。
超过50个OSD，可以参考pgcalc计算
ceph osd pool create mypool 128 128 #创建存储池  默认三副本
rbd pool init mypool #初始化存储池
rbd create mypool/disk01 --size 10G  #创建存储盘
sudo rbd map disk01  #内核版本低无法挂载需要升级内核或者
rbd feature disable mypool/disk01 exclusive-lock object-map fast-diff deep-flatten
sudo rbd map disk01  #直接映射本地 
或者
rbd create mypool/disk02 --size 10G --image-feature layering #指定参数 10G存储盘
rbd ls -l #查看
sudo rbd map disk02  #直接映射本地 

sudo mkfs.xfs /dev/rbd0 #格式化
sudo mount /dev/rbd0 /mnt #挂载
[cephadm@cephadm ceph-cluster]$ df -h
/dev/rbd0                 10G   33M   10G    1% /mnt

centos7 升级内核

http://elrepo.org/tiki/HomePage

导入key 安装yum源
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm
查看可以安装内核版本
yum --disablerepo="*" --enablerepo="elrepo-kernel" list available
安装指定内核版本
yum --enablerepo=elrepo-kernel install kernel-ml kernel-ml-devel
查看当前内核
awk -F\' ' $1=="menuentry " {print $2}' /etc/grub2.cfg
CentOS Linux (5.11.11-1.el7.elrepo.x86_64) 7 (Core)
CentOS Linux (3.10.0-1127.el7.x86_64) 7 (Core)
CentOS Linux (0-rescue-d7999659ea104b01a700c01370e679df) 7 (Core)
设置内核从5.11.11
grub2-set-default 0
reboot
uname -r
卸载低版本内核
yum remove kernel

sudo rbd create rbd/disk03 --size 2G
sudo rbd map disk03
sudo mkfs.xfs /dev/rbd0
sudo mount /dev/rbd0 /mnt

rbd块扩容 不建议rbd块中分区

rbd resize rbd/disk03 --size 5G  扩容5G
rbd -p rbd info --image disk03 查看磁盘信息
xfs_growfs /dev/rbd0 xfs扩容磁盘
resize2fs /dev/rbd0 ext系列扩展磁盘

3.2 挂载cephfs文件系统

https://docs.ceph.com/en/latest/man/8/mount.ceph/

创建名为cephfs_data的pool
ceph osd pool create cephfs_data 128
创建名为cephfs_metadata的pool
ceph osd pool create cephfs_metadata 128
启用
ceph fs new cephfs cephfs_metadata cephfs_data 
ceph fs ls
ceph mds stat 
[root@stor04 mnt]# cat /etc/ceph/ceph.client.admin.keyring 
[client.admin]
    key = AQB4NGRg1aThJxAAE1h/AKBKfnjIL6eOsUh8QA==
    caps mds = "allow *"
    caps mgr = "allow *"
    caps mon = "allow *"
    caps osd = "allow *"
客户端安装：
yum -y install ceph-fuse
sudo mount -t ceph 10.0.0.52:6789:/ /mnt -o name=admin,secret=AQB4NGRg1aThJxAAE1h/AKBKfnjIL6eOsUh8QA==
或者
sudo ceph-authtool -p /etc/ceph/ceph.client.admin.keyring > admin.key 
chmod 600 admin.key
mount -t ceph mon01:6789:/ /mnt -o name=admin,secretfile=admin.key 
[root@stor04 mnt]# df -h
10.0.0.52:6789:/          100G     0   100G    0% /mnt

4. 删除ceph pool

[cephadm@cephadm ceph-cluster]$ vim ceph.conf  ##添加可以删除pool参数
[global]
mon_allow_pool_delete = true
[cephadm@cephadm ceph-cluster]$ ceph-deploy --overwrite-conf admin cephadm mon01 mon02 mon03 stor04 #下发修改后ceph文件
重启4个osd节点的mon mgr 服务
systemctl list-units --type=service|grep ceph 查看服务
systemctl restart ceph-mgr@mon01.service
systemctl restart ceph-mon@mon01.service
systemctl restart ceph-mgr@mon02.service
systemctl restart ceph-mon@mon02.service
systemctl restart ceph-mgr@mon03.service
systemctl restart ceph-mon@mon03.service
systemctl restart ceph-mgr@stor04.service
systemctl restart ceph-mon@stor04.service
删除pool pool名字写两遍 挂载的无法删除 显示busy
ceph osd pool rm mypool mypool --yes-i-really-really-mean-it