清除ceph原有配置

清除安装包
[cephuser@ceph-admin ~]$ ceph-deploy purge ceph1 ceph2 ceph3

清除配置信息

[cephuser@ceph-admin ~]$ ceph-deploy purgedata ceph1 ceph2 ceph3
[cephuser@ceph-admin ~]$ ceph-deploy forgetkeys

每个节点删除残留的配置文件

sudo rm -rf /var/lib/ceph/osd/*
sudo rm -rf /var/lib/ceph/mon/*
sudo rm -rf /var/lib/ceph/mds/*
sudo rm -rf /var/lib/ceph/bootstrap-mds/*
sudo rm -rf /var/lib/ceph/bootstrap-osd/*
sudo rm -rf /var/lib/ceph/bootstrap-mon/*
sudo rm -rf /var/lib/ceph/tmp/*
sudo rm -rf /etc/ceph/*
sudo rm -rf /var/run/ceph/*

文档资料

https://blog.csdn.net/weixin_45529004/article/details/108201670

1、环境准备
192.168.171.11 admin
192.168.171.12 node1
192.168.171.13 node2
2、初始化环境

#主机名配置->每个节点执行

hostnamectl set-hostname admin
hostnamectl set-hostname node1
hostnamectl set-hostname node2

#配置hosts->每个节点执行

cat <<"EOF">/etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.171.11    admin 
192.168.171.12    node1 
192.168.171.13    node2
EOF

#增加用户->每个节点执行

useradd -d /home/ceph -m ceph
1
echo myceph | passwd –stdin ceph
1
echo “ceph ALL = (root) NOPASSWD:ALL” >> /etc/sudoers.d/ceph
1
chmod 0440 /etc/sudoers.d/ceph
1

#安装ntp->每个节点执行

yum install ntp ntpdate ntp-doc -y
systemctl start ntpd
systemctl status ntpd
systemctl enable ntpd

#安装epel源和python-pip->管理节点(admin)

sudo yum -y install epel-release
sudo yum install python-pip

#配置ceph相关yum源->每个节点执行

cat <<END >/etc/yum.repos.d/ceph.repo
[Ceph]
name=Ceph packages for \$basearch
baseurl=http://mirrors.aliyun.com/ceph/rpm-nautilus/el7/\$basearch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc

[Ceph-noarch]
name=Ceph noarch packages
baseurl=http://mirrors.aliyun.com/ceph/rpm-nautilus/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc

[ceph-source]
name=Ceph source packages
baseurl=http://mirrors.aliyun.com/ceph/rpm-nautilus/el7/SRPMS
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
END

#关闭selinx和防火墙->每个节点执行

setenforce 0
sed -i "s/SELINUX=enforcing/SELINUX=permissive/g" /etc/selinux/config
systemctl disable firewalld.service;systemctl stop firewalld.service

#配置免密码登录->每个节点执行

su - ceph
ssh-keygen
ssh-copy-id ceph@admin
ssh-copy-id ceph@node1
ssh-copy-id ceph@node2
cat <<END >~/.ssh/config 
Host admin
   Hostname admin
   User ceph
Host node1
   Hostname node1
   User ceph
Host node2
   Hostname node2
   User ceph
END

#解决ceph-deploy new的时候报错Bad owner or permissions on /home/ceph/.ssh/config->每个节点执行

sudo chmod 600 /home/ceph/.ssh/config

2、安装ceph-deploy

#->管理节点执行

sudo yum install ceph-deploy -y

[ceph@admin ~]$ ceph-deploy –version
2.0.1

#或者
sudo rpm -Uvh –replacepkgs http://mirrors.163.com/ceph/rpm-nautilus/el7/noarch/ceph-release-1-1.el7.noarch.rpm
sudo rpm -ivh http://mirrors.163.com/ceph/rpm-nautilus/el7/noarch/ceph-deploy-2.0.1-0.noarch.rpm –nodeps

#创建存放ceph配置文件夹->管理节点执行

sudo mkdir /etc/ceph && sudo chown -R ceph.ceph /etc/ceph/

3、创建mon节点

#->管理节点执行

#创建多个mon节点互备,也可以只创建一个节点

cd /etc/ceph
ceph-deploy new --cluster-network 192.168.2.0/24 --public-network 192.168.2.0/24 admin node1 node2
[ceph@admin ceph]$ ll
total 12
-rw-rw-r-- 1 ceph ceph  221 May  3 23:51 ceph.conf
-rw-rw-r-- 1 ceph ceph 3158 May  3 23:51 ceph-deploy-ceph.log
-rw------- 1 ceph ceph   73 May  3 23:51 ceph.mon.keyring

#修改配置文件

cd /etc/ceph

#在[global]标签下,添加下面一行:

echo "osd pool default size = 2" >> ceph.conf

4、安装ceph软件包

#->管理节点执行

cd /etc/ceph
ceph-deploy install admin node1 node2

#或者
ceph-deploy install admin node1 node2 –repo-url http://mirrors.163.com/ceph/rpm-nautilus/el7/ –gpg-url http://mirrors.163.com/ceph/keys/release.asc

#安装完成后每个节点都会生成/etc/ceph文件夹,但是目录的用户和组变成了root,此时需要调整为ceph->每个节点执行

sudo chown -R ceph.ceph /etc/ceph

#小技巧->每个节点单独安装ceph相关软件包速度比较快,如果报错还是需要运行一下install

sudo yum -y install epel-release
sudo yum -y install ceph ceph-radosgw
sudo yum install ceph-mgr-dashboard -y

5、初始化ceph mon节点

#->管理节点执行

#初始化ceph monitor node 后在当前目录会产生一些*.keyring,这是ceph组件间进行安全访问时所需要的

cd /etc/ceph

#每个节点执行,否则会报错运行ceph-deploy mon create-initial会报错,上个步骤有说明

sudo chown -R ceph.ceph /etc/ceph
[ceph_deploy][ERROR ] IOError: [Errno 13] Permission denied: 'ceph.client.admin.keyring'
ceph-deploy mon create-initial
[ceph@admin ceph]$ ll
-rw-------. 1 ceph ceph    113 8月  24 15:15 ceph.bootstrap-mds.keyring
-rw-------. 1 ceph ceph    113 8月  24 15:15 ceph.bootstrap-mgr.keyring
-rw-------. 1 ceph ceph    113 8月  24 15:15 ceph.bootstrap-osd.keyring
-rw-------. 1 ceph ceph    113 8月  24 15:15 ceph.bootstrap-rgw.keyring
-rw-------. 1 ceph ceph    151 8月  24 15:15 ceph.client.admin.keyring
-rw-r--r--. 1 root root    336 8月  24 15:19 ceph.conf
-rw-rw-r--. 1 ceph ceph 345820 8月  24 15:30 ceph-deploy-ceph.log
-rw-------. 1 ceph ceph     73 8月  24 15:04 ceph.mon.keyring
-rw-r--r--. 1 ceph ceph     92 8月  11 04:44 rbdmap

#->每个节点执行
ps -ef | grep ceph-mon # 检测ceph-mon是否已经运行起来

[ceph@node1 ~]$ ps -ef | grep ceph-mon
ceph 3591 1 0 15:55 ? 00:00:08 /usr/bin/ceph-mon -f –cluster ceph –id node1 –setuser ceph –setgroup ceph

#执行完这一步所有节点的/etc/ceph.conf的用户和组变成了root

[ceph@admin ceph]$ ll /etc/ceph/ceph.conf
-rw-r--r--. 1 root root 336 824 15:19 /etc/ceph/ceph.con
[root@node1 ~]# ll /etc/ceph/ceph.conf
-rw-r--r-- 1 root root 336 824 15:30 /etc/ceph/ceph.conf
[root@node2 ~]# ll /etc/ceph/ceph.conf
-rw-r--r-- 1 root root 336 824 15:30 /etc/ceph/ceph.conf

6、推送配置文件

#配置admin key到每个节点->管理节点执行

cd /etc/ceph
ceph-deploy admin node1 node2

#执行完后,每个节点会多出用户和组为root的ceph.client.admin.keyring文件

[root@node1 ~]# ll /etc/ceph/
总用量 12
-rw------- 1 ceph ceph 151 8月  24 15:16 ceph.client.admin.keyring
-rw-r--r-- 1 root root 336 8月  24 15:30 ceph.conf
-rw-r--r-- 1 ceph ceph  92 8月  11 04:44 rbdmap
-rw------- 1 ceph ceph   0 8月  24 15:14 tmpfPzTmR
[root@node2 ~]# ll /etc/ceph/
总用量 12
-rw------- 1 ceph ceph 151 8月  24 15:16 ceph.client.admin.keyring
-rw-r--r-- 1 root root 336 8月  24 15:30 ceph.conf
-rw-r--r-- 1 ceph ceph  92 8月  11 04:44 rbdmap
-rw------- 1 ceph ceph   0 8月  24 15:14 tmpoKI_et

#其它节点执行ceph -s报错处理,如果用root用户执行不会报错

[ceph@node1 ceph]$ ceph -s
[errno 2] error connecting to the cluster
[ceph@node1 ceph]$ sudo chown -R ceph.ceph /etc/ceph
[ceph@node1 ceph]$ ceph -s
  cluster:
    id:     faa6ec21-90d8-4ee2-928c-4a631423e764
    health: HEALTH_OK

  services:
    mon: 3 daemons, quorum admin,node1,node2 (age 12m)
    mgr: no daemons active
    osd: 0 osds: 0 up, 0 in

  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0 B used, 0 B / 0 B avail
    pgs:  

7、创建mgr节点并启用dashboard

#创建mgr节点->管理节点执行

ceph-deploy mgr create admin node1 node2

#检测ceph-mgr是否已经运行起来->每个节点执行

ps -ef | grep ceph-mgr
ceph        4482       1  0 16:03 ?        00:00:12 /usr/bin/ceph-mgr -f --cluster ceph --id node1 --setuser ceph --setgroup ceph

#->管理节点执行

[ceph@admin ceph]$ ceph mgr module enable dashboard
Error ENOENT: all mgr daemons do not support module 'dashboard', pass --force to force enablement

#避免出现上述Error ENOENT,需要为每个节点安装ceph-mgr-dashboard->每个节点执行

sudo yum install ceph-mgr-dashboard -y

#再次在admin节点执行

ceph mgr module enable dashboard

#确认开启mgr模块->每个节点执行

ceph mgr module ls | less

#查看mgr/dashboard相关参数,这一步有值的话,可以进行配置->每个节点执行

ceph config ls | grep mgr/dashboard

#配置dashboard相关->每个节点执行

ceph config set mgr mgr/dashboard/server_addr 0.0.0.0
ceph config set mgr mgr/dashboard/server_port 7000
ceph config set mgr mgr/dashboard/ssl false

#删除配置->每个节点执行

ceph config rm mgr mgr/dashboard/server_addr
ceph config rm mgr mgr/dashboard/server_port
ceph config rm mgr mgr/dashboard/ssl

#确认dashboard服务开启->每个节点执行

ceph mgr services
{
    "dashboard": "http://0.0.0.0:7000/"
}

#查看配置的最近10条日志,+代表新增配置,-代表删除配置->每个节点执行

ceph config log 10

#创建该地址http://192.168.171.11:7000的访问用户密码

#注意访问地址以ceph -s 显示的active mgr节点地址为主,不一定为192.168.171.11

ceph dashboard ac-user-create cephuser cephpwd administrator

{“username”: “cephuser”, “lastUpdate”: 1556949263, “name”: null, “roles”: [“administrator”], “password”: “$2b12 1212/jh6ww8qXVcRYXfw4x9LkOQk7WXORIfig1/5ikA5FBNhai2fXGNQG”, “email”: null}

#访问页面的时候报500错误,通过查看/var/log/ceph/ceph-mgr.admin.log日志出现以下错误,重启三个节点就恢复正常了

2020-08-24 15:03:39.244 7f869b27e700 0 mgr[dashboard] [192.168.171.1:55174] [GET] [500] [0.004s] [cephuser] [1.3K] /api/summary
2020-08-24 15:03:39.244 7f869b27e700 0 mgr[dashboard] [’{“status”: “500 Internal Server Error”, “version”: “3.2.2”, “detail”: “The server encountered an unexpected condition which prevented it from fulfilling the request.”, “traceback”: “Traceback (most recent call last):\n File \”/usr/lib/python2.7/site-packages/cherrypy/_cprequest.py\", line 656, in respond\n response.body = self.handler()\n File \"/usr/lib/python2.7/site-packages/cherrypy/lib/encoding.py\", line 188, in call\n self.body = self.oldhandler(*args, **kwargs)\n File \"/usr/lib/python2.7/site-packages/cherrypy/_cptools.py\", line 221, in wrap\n return self.newhandler(innerfunc, *args, **kwargs)\n File \"/usr/share/ceph/mgr/dashboard/services/exception.py\", line 88, in dashboard_exception_handler\n return handler(*args, **kwargs)\n File \"/usr/lib/python2.7/site-packages/cherrypy/_cpdispatch.py\", line 34, in call\n return self.callable(*self.args, **self.kwargs)\n File \"/usr/share/ceph/mgr/dashboard/controllers/init.py\", line 649, in inner\n ret = func(*args, **kwargs)\n File \"/usr/share/ceph/mgr/dashboard/controllers/summary.py\", line 79, in call\n ‘mgr_host’: self._get_host(),\n File \"/usr/share/ceph/mgr/dashboard/controllers/summary.py\", line 68, in _get_host\n return services[‘dashboard’]\nKeyError: ‘dashboard’\n"}’]

8、创建osd

#增加osd 无需对/dev/sdb格式化->管理节点执行

ceph-deploy osd create node1 --data /dev/sdb
ceph-deploy osd create node2 --data /dev/sdb

#确认增加成功

[ceph@admin ceph]$ ceph -s
  cluster:
    id:     5080e921-896a-4e7c-88ba-afff843a7079
    health: HEALTH_OK

  services:
    mon: 1 daemons, quorum node1 (age 9h)
    mgr: node1(active, since 8h), standbys: admin, node2
    osd: 2 osds: 2 up (since 97s), 2 in (since 97s)

  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   2.0 GiB used, 12 GiB / 14 GiB avail
    pgs:  

9、至此ceph集群部署完成,后面就是创建块存储、对象存储等

创建应用网关

 ceph-deploy rgw create admin 

The Ceph Object Gateway (RGW) default port 7480

相关资料

https://www.cnblogs.com/passzhang/p/12151835.html

文档更新时间: 2020-10-31 00:58   作者:月影鹏鹏