干货—— ES 备份恢复
转自：https://www.cnblogs.com/sxy-blog/p/18128248
1.官网提供snap快照备份恢复
https://www.elastic.co/guide/en/elasticsearch/reference/7.9/snapshot-restore.html
环境要求
要求所有es节点，挂载同一个共享目录，可以使用NFS。
注意背景
[root@devops01 ~]#id elasticsearch
uid=998(elasticsearch) gid=996(elasticsearch) groups=996(elasticsearch)
[root@devops01 ~]#搭建NFS机器，服务端
yum install nfs-utils -y
# 根据es的用户信息，创建挂载用户
groupadd elasticsearch -g 996
useradd elasticsearch -g 996 -u 998 -M -s /sbin/nologin
cat > /etc/exports << 'EOF'
/es-nfs-data 10.0.0.0/24(rw,sync,all_squash,anonuid=998,anongid=996)
EOF
# 目录创建授权
[root yc-k8s ~]#mkdir -p /es-nfs-data
[root yc-k8s ~]#chown -R elasticsearch.elasticsearch /es-nfs-data/
[root yc-k8s ~]#ll -d /es-nfs-data/
drwxr-xr-x. 2 elasticsearch elasticsearch 6 Dec 3 02:16 /es-nfs-data/
[root yc-k8s ~]#
[root yc-k8s ~]#systemctl restart nfs
[root yc-k8s ~]#showmount -e 10.0.0.122
Export list for 10.0.0.122:
/es-nfs-data 10.0.0.0/24es节点安装nfs插件
cat > nfs-client.sh <<'EOF'
yum install nfs-utils -y
mkdir /es-client-data -p
mount -t nfs 10.0.0.122:/es-nfs-data /es-client-data
EOF
sh nfs-client.sh
# 检查挂载
df -h |grep es-client-dataes节点修改config，开启snapshot
每个节点config.yml添加配置
path.repo: /es-client-data/
# 如
[es-node3 root ~]#cat /etc/elasticsearch/elasticsearch.yml
cluster.name: yuchao_es
node.name: es-node3
path.data: /var/lib/elasticsearch/
path.logs: /var/log/elasticsearch/
bootstrap.memory_lock: true
network.host: 127.0.0.1,10.0.0.20
http.port: 9200
discovery.seed_hosts: ["10.0.0.18","10.0.0.19","10.0.0.20"]
cluster.initial_master_nodes: ["10.0.0.18"]
path.repo: /es-client-data/
[es-node3 root ~]#
# 重启
systemctl restart elasticsearch.service确保重启正确
1.先注册快照仓库
PUT /_snapshot/my_backup
{
"type": "fs",
"settings": {
"location": "/es-client-data/my_backup_location",
"compress": true
}
}
# 可以查询仓库信息
GET /_snapshot/my_backup
2.创建一个快照
PUT /_snapshot/my_backup/snapshot_1?wait_for_completion=true
检查快照
图解nfs快照
3.针对具体index的快照
# 创建第二个快照，名字snapshot_2
PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=true
{
"indices": "t1,t2",
"ignore_unavailable": true,
"include_global_state": false
}
}
4.查看快照信息
GET /_snapshot
GET /_snapshot/my_backup/
GET /_snapshot/my_backup/snapshot_1
GET /_snapshot/my_backup/snapshot_2
5.查看正在运行的快照
GET /_snapshot/my_backup/_current
{
"snapshots" : [ ]
}6.演练恢复索引
1. 删除 t2库
2.基于有该t2 库的快照
POST /_snapshot/my_backup/snapshot_2/_restore
3.默认恢复，发现有同名index，导致恢复错误
4.可以单独还原某个index，或者删除es数据，全库恢复。
GET /_snapshot
GET /_snapshot/my_backup/
GET /_snapshot/my_backup/snapshot_1
GET /_snapshot/my_backup/snapshot_2
GET /_snapshot/my_backup/_current
POST /_snapshot/my_backup/snapshot_2/_restore
# 恢复指定的索引
POST /_snapshot/my_backup/snapshot_2/_restore
{
"indices": "t2",
"ignore_unavailable": true,
"include_global_state": false,
"rename_pattern": "index_(.+)",
"rename_replacement": "restored_index_$1",
"include_aliases": false
}
# 参数解释
上面的indices, 表示只恢复索引’index_1’
rename_pattern: 表示重命名索引以’index_’开头的索引.（注意我们这里索引叫做t2，所以没作用）
rename_replacement: 表示将所有的索引重命名为’restored_index_xxx’.如index_1会被重命名为restored_index_1.
指定index恢复
恢复且修改index名
# 如果想修改恢复后的index名字
POST /_snapshot/my_backup/snapshot_2/_restore
{
"indices": "t2",
"ignore_unavailable": true,
"include_global_state": false,
"rename_pattern": "t(.+)",
"rename_replacement": "restored_index_$1",
"include_aliases": false
}
7.用日期命名快照
不建议去用。
PUT /_snapshot/my_backup/<snapshot-{now/d}>
PUT /_snapshot/my_backup/%3Csnapshot-%7Bnow%2Fd%7D%3E
8.记录es练习接口
PUT /_snapshot/my_backup
{
"type": "fs",
"settings": {
"location": "/es-client-data/my_backup_location",
"compress": true
}
}
GET /_snapshot/my_backup
PUT /_snapshot/my_backup/snapshot_1?wait_for_completion=true
PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=true
{
"indices": "t1,t2",
"ignore_unavailable": true,
"include_global_state": false
}
}
GET /_snapshot/
GET /_snapshot/my_backup/
GET /_snapshot/my_backup/_all
GET /_snapshot/my_backup/snapshot_1
GET /_snapshot/my_backup/snapshot_2
GET /_snapshot/my_backup/_current
POST /_snapshot/my_backup/snapshot_2/_restore
POST /_snapshot/my_backup/snapshot_2/_restore
{
"indices": "t2",
"ignore_unavailable": true,
"include_global_state": false,
"rename_pattern": "index_(.+)",
"rename_replacement": "restored_index_$1",
"include_aliases": false
}
POST /_snapshot/my_backup/snapshot_2/_restore
{
"indices": "t2",
"ignore_unavailable": true,
"include_global_state": false,
"rename_pattern": "t(.+)",
"rename_replacement": "restored_index_$1",
"include_aliases": false
}
PUT /_snapshot/my_backup/%3Csnapshot-%7Bnow%2Fd%7D%3E2.第三方备份工具
1.安装node环境，基于nodejs开发的备份工具
# 注意node版本要求和于超老师一致
2.软件官网
https://www.npmjs.com/package/elasticdump2.1 安装node
wget https://nodejs.org/dist/v10.16.3/node-v10.16.3-linux-x64.tar.xz
tar -xf node-v10.16.3-linux-x64.tar.xz
[root yc-k8s /opt]#
[root yc-k8s /opt]#ln -s node-v10.16.3-linux-x64/ node
[root yc-k8s /opt]#
[root yc-k8s /opt]#echo 'export PATH=/opt/node/bin:$PATH' >> /etc/profile
[root yc-k8s /opt]#
[root yc-k8s /opt]#source /etc/profile
[root yc-k8s /opt]#
[root yc-k8s /opt]#node -v
v10.16.3
[root yc-k8s /opt]#
[root yc-k8s /opt]#npm -v
6.9.0
[root yc-k8s /opt]#
# 设置淘宝源
# 于超老师这里测试，腾讯源快一点
npm config set registry https://registry.npm.taobao.org
npm config set registry http://mirrors.cloud.tencent.com/npm/
npm config get registry
# 安装elasticdump备份工具
[root yc-k8s /opt]#npm install elasticdump -g
[root yc-k8s /opt]#elasticdump --version
6.94.12.2 备份命令
备份es数据为可读的json文件
[root yc-k8s /opt]#elasticdump \
> --input=http://10.0.0.18:9200/t1 \
> --output=/es-nfs-data/t1.json \
> --type=data
Sat, 03 Dec 2022 14:35:45 GMT | starting dump
Sat, 03 Dec 2022 14:35:45 GMT | got 8 objects from source elasticsearch (offset: 0)
Sat, 03 Dec 2022 14:35:45 GMT | sent 8 objects to destination file, wrote 8
Sat, 03 Dec 2022 14:35:45 GMT | got 0 objects from source elasticsearch (offset: 8)
Sat, 03 Dec 2022 14:35:45 GMT | Total Writes: 8
Sat, 03 Dec 2022 14:35:45 GMT | dump complete
[root yc-k8s /opt]#
[root yc-k8s /es-nfs-data]#cat t1.json
{"_index":"t1","_type":"_doc","_id":"7HNg1IQBRcFJVNOeWVxR","_score":1,"_source":{"name":"yu1","age":"28","address":"JS","job":"dev"}}
{"_index":"t1","_type":"_doc","_id":"7XNg1IQBRcFJVNOeWVyL","_score":1,"_source":{"name":"yu2","age":"27","address":"BJ","job":"dev"}}
{"_index":"t1","_type":"_doc","_id":"7nNg1IQBRcFJVNOeWVyd","_score":1,"_source":{"name":"yu3","age":"26","address":"SD","job":"ops"}}
{"_index":"t1","_type":"_doc","_id":"73Ng1IQBRcFJVNOeWVyq","_score":1,"_source":{"name":"yu4","age":"25","address":"JX","job":"ops"}}
{"_index":"t1","_type":"_doc","_id":"8HNg1IQBRcFJVNOeWVy5","_score":1,"_source":{"name":"jack01","age":"19","address":"YN","job":"test"}}
{"_index":"t1","_type":"_doc","_id":"8XNg1IQBRcFJVNOeWVzD","_score":1,"_source":{"name":"tom02","age":"30","address":"DB","job":"test"}}
{"_index":"t1","_type":"_doc","_id":"8nNg1IQBRcFJVNOeWVzN","_score":1,"_source":{"name":"david03","age":"30","address":"BJ","job":"test"}}
{"_index":"t1","_type":"_doc","_id":"83Ng1IQBRcFJVNOeWVzY","_score":1,"_source":{"name":"xiaohei01","age":"17","address":"BJ","job":"ops"}}
[root yc-k8s /es-nfs-data]#备份且压缩
elasticdump \
--input=http://10.0.0.18:9200/t2 \
--output=$ \
| gzip > /es-nfs-data/t2.json.gz
[root yc-k8s /es-nfs-data]#gzip -d t2.json.gz
[root yc-k8s /es-nfs-data]#ll
total 12
drwxr-xr-x. 3 elasticsearch elasticsearch 4096 Dec 3 04:50 my_backup_location
-rw-r--r--. 1 root root 1090 Dec 3 22:35 t1.json
-rw-r--r--. 1 root root 402 Dec 3 22:37 t2.json
[root yc-k8s /es-nfs-data]#cat t2.json
{"_index":"t2","_type":"_doc","_id":"9HNg1IQBRcFJVNOe-1xE","_score":1,"_source":{"name":"yu1","age":"28","address":"JS","job":"dev"}}
{"_index":"t2","_type":"_doc","_id":"9XNg1IQBRcFJVNOe-1yA","_score":1,"_source":{"name":"yu2","age":"27","address":"BJ","job":"dev"}}
{"_index":"t2","_type":"_doc","_id":"9nNg1IQBRcFJVNOe-1yS","_score":1,"_source":{"name":"yu3","age":"26","address":"SD","job":"ops"}}
[root yc-k8s /es-nfs-data]#2.3 恢复数据
# 恢复数据，就是input，output反过来
[root yc-k8s /es-nfs-data]#elasticdump \
> --input=/es-nfs-data/t2.json \
> --output=http://10.0.0.18:9200/t2
Sat, 03 Dec 2022 14:44:23 GMT | starting dump
Sat, 03 Dec 2022 14:44:23 GMT | got 3 objects from source file (offset: 0)
Sat, 03 Dec 2022 14:44:23 GMT | sent 3 objects to destination elasticsearch, wrote 3
Sat, 03 Dec 2022 14:44:23 GMT | got 0 objects from source file (offset: 3)
Sat, 03 Dec 2022 14:44:23 GMT | Total Writes: 3
Sat, 03 Dec 2022 14:44:23 GMT | dump complete
[root yc-k8s /es-nfs-data]#2.4 批量备份
# 去掉以点开头的index
curl -s 10.0.0.18:9200/_cat/indices | awk '{print $3}' | grep -v '^\.'
# 脚本
#!/bin/bash
# author : www.yuchaoit.cn
indexs=$(curl -s 10.0.0.18:9200/_cat/indices | awk '{print $3}' | grep -v '^\.')
for i in $indexs
do
elasticdump \
--input=http://10.0.0.18:9200/${i} \
--output=/es-nfs-data/${i}.json \
--type=data
done备份结果
root yc-k8s /es-nfs-data]#
[root yc-k8s /es-nfs-data]#sh for-index.sh
Sat, 03 Dec 2022 14:54:57 GMT | starting dump
Sat, 03 Dec 2022 14:54:57 GMT | got 3 objects from source elasticsearch (offset: 0)
Sat, 03 Dec 2022 14:54:57 GMT | sent 3 objects to destination file, wrote 3
Sat, 03 Dec 2022 14:54:57 GMT | got 0 objects from source elasticsearch (offset: 3)
Sat, 03 Dec 2022 14:54:57 GMT | Total Writes: 3
Sat, 03 Dec 2022 14:54:57 GMT | dump complete
Sat, 03 Dec 2022 14:54:57 GMT | starting dump
Sat, 03 Dec 2022 14:54:57 GMT | got 8 objects from source elasticsearch (offset: 0)
Sat, 03 Dec 2022 14:54:57 GMT | sent 8 objects to destination file, wrote 8
Sat, 03 Dec 2022 14:54:57 GMT | got 0 objects from source elasticsearch (offset: 8)
Sat, 03 Dec 2022 14:54:57 GMT | Total Writes: 8
Sat, 03 Dec 2022 14:54:57 GMT | dump complete
Sat, 03 Dec 2022 14:54:57 GMT | starting dump
Sat, 03 Dec 2022 14:54:57 GMT | got 3 objects from source elasticsearch (offset: 0)
Sat, 03 Dec 2022 14:54:57 GMT | sent 3 objects to destination file, wrote 3
Sat, 03 Dec 2022 14:54:57 GMT | got 0 objects from source elasticsearch (offset: 3)
Sat, 03 Dec 2022 14:54:57 GMT | Total Writes: 3
Sat, 03 Dec 2022 14:54:57 GMT | dump complete
[root yc-k8s /es-nfs-data]#ll
\total 20
-rw-r--r--. 1 root root 248 Dec 3 22:54 for-index.sh
drwxr-xr-x. 3 elasticsearch elasticsearch 4096 Dec 3 04:50 my_backup_location
-rw-r--r--. 1 root root 444 Dec 3 22:54 restored_index_2.json
-rw-r--r--. 1 root root 1090 Dec 3 22:54 t1.json
-rw-r--r--. 1 root root 402 Dec 3 22:54 t2.json
[root yc-k8s /es-nfs-data]#2.5 密码认证
es有密码时，备份命令
elasticdump \
--input=http://name:[email protected]:9200/t2 \
--output=/es-nfs-data/t2.json \
--type=data2.6 建议用法
1. 想分析es数据，用elasticdump导出json
2. 仅仅备份数据，用es官网工具
3. 恢复es数据，如果数据冲突会被覆盖3.es安全认证
逻辑图
官网资料
https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-stack-security.html
1.创建证书
1. 创建证书，输入密码
[root@es-node1 ~]#
[root@es-node1 ~]#
[root@es-node1 ~]#/usr/share/elasticsearch/bin/elasticsearch-certutil ca
This tool assists you in the generation of X.509 certificates and certificate
signing requests for use with SSL/TLS in the Elastic stack.
The 'ca' mode generates a new 'certificate authority'
This will create a new X.509 certificate and private key that can be used
to sign certificate when running in 'cert' mode.
Use the 'ca-dn' option if you wish to configure the 'distinguished name'
of the certificate authority
By default the 'ca' mode produces a single PKCS#12 output file which holds:
* The CA certificate
* The CA's private key
If you elect to generate PEM format certificates (the -pem option), then the output will
be a zip file containing individual files for the CA certificate and private key
Please enter the desired output file [elastic-stack-ca.p12]:
Enter password for elastic-stack-ca.p12 :
[root@es-node1 ~]#
[root@es-node1 ~]#
[root@es-node1 ~]#
[root@es-node1 ~]#file /usr/share/elasticsearch/elastic-stack-ca.p12
/usr/share/elasticsearch/elastic-stack-ca.p12: data
2.根据ca整数，创建私钥，也可以二次设置密码
/usr/share/elasticsearch/bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12
3.检查
[root@es-node1 /usr/share/elasticsearch]#ll
total 576
drwxr-xr-x 2 root root 4096 Nov 18 23:53 bin
-rw------- 1 root root 3443 Dec 3 23:10 elastic-certificates.p12
-rw------- 1 root root 2527 Dec 3 23:08 elastic-stack-ca.p12
drwxr-xr-x 9 root root 107 Nov 18 23:53 jdk
drwxr-xr-x 3 root root 4096 Nov 18 23:53 lib
-rw-r--r-- 1 root root 13675 Sep 2 2020 LICENSE.txt
drwxr-xr-x 52 root root 4096 Nov 18 23:53 modules
-rw-rw-r-- 1 root root 544318 Sep 2 2020 NOTICE.txt
drwxr-xr-x 2 root root 6 Sep 2 2020 plugins
-rw-r--r-- 1 root root 7007 Sep 2 2020 README.asciidoc
4.复制证书到固定位置，同步es节点
[root@es-node1 /usr/share/elasticsearch]#scp -r /etc/elasticsearch/certs [email protected]:/etc/elasticsearch/
elastic-certificates.p12 100% 3443 5.1MB/s 00:00
elastic-stack-ca.p12 100% 2527 3.1MB/s 00:00
[root@es-node1 /usr/share/elasticsearch]#scp -r /etc/elasticsearch/certs [email protected]:/etc/elasticsearch/
elastic-certificates.p12 100% 3443 5.3MB/s 00:00
elastic-stack-ca.p12 100% 2527 2.8MB/s 00:00
[root@es-node1 /usr/share/elasticsearch]#
5.修改es节点配置文件，开启安全功能
# es安全组件，xpack，开启即可。
# 来自于官网的教程，复制粘贴即可，有机会再阅读es数据，逐步深入学习。
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: certs/elastic-stack-ca.p12
xpack.security.transport.ssl.truststore.path: certs/elastic-stack-ca.p12
6.注意修改es目录权限，重启进程
chown -R elasticsearch.elasticsearch /etc/elasticsearch/
systemctl restart elasticsearch.service
7.检查es状态
netstat -tunlp|grep 9200此时就已连不上了，需要账密了
创建账户，密码
默认会给一些列的内置用户，创建密码，统一123123
[root@es-node1 ~]#/usr/share/elasticsearch/bin/elasticsearch-setup-passwords interactive
Initiating the setup of passwords for reserved users elastic,apm_system,kibana,kibana_system,logstash_system,beats_system,remote_monitoring_user.
You will be prompted to enter passwords as the process progresses.
Please confirm that you would like to continue [y/N]y
Enter password for [elastic]:
passwords must be at least [6] characters long
Try again.
Enter password for [elastic]:
Reenter password for [elastic]:
Enter password for [apm_system]:
Reenter password for [apm_system]:
Enter password for [kibana_system]:
Reenter password for [kibana_system]:
Enter password for [logstash_system]:
Reenter password for [logstash_system]:
Enter password for [beats_system]:
Reenter password for [beats_system]:
Enter password for [remote_monitoring_user]:
Reenter password for [remote_monitoring_user]:
Changed password for user [apm_system]
Changed password for user [kibana_system]
Changed password for user [kibana]
Changed password for user [logstash_system]
Changed password for user [beats_system]
Changed password for user [remote_monitoring_user]
Changed password for user [elastic]内置角色解释
账户elastic为elasticsearch超级管理员，拥有所有权限
账户kibana用于kibana组件获取相关信息用于web展示
账户logstash_system用于logstash服务获取elasticsearch的监控数据
账户beats_system用于存储es监控信息时用
账户kibana_system，用于kibana安全与es集群通讯用。
es官网，内置角色
https://www.elastic.co/guide/en/elasticsearch/reference/7.9/built-in-roles.html2.创建kibana登录用户
务必注意，这里kibana使用刚才es创建的 账户，密码，来连接es集群。
密码保证一致，否则kibana，无法和es通信了。
[root@es-node1 ~]#vim /etc/kibana/kibana.yml
42 # If your Elasticsearch is protected with basic authentication, these settings provide
43 # the username and password that the Kibana server uses to perform maintenance on the Kibana
44 # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
45 # is proxied through the Kibana server.
46 elasticsearch.username: "kibana_system"
47 elasticsearch.password: "123123"
重启kibana
[root@es-node1 ~]#systemctl restart kibanakibana正确安全连接es
elastic
123123
普通用户kibana 123123 也可以登录es-head登录认证
3.创建用户
kibana先创建es的索引
便于查询数据。
kibana功能space
需求：1.限制开发小于，登录kibana，只能使用某个功能，如discover,且只能看t2 索引
实践：
1. 创建space ，dev，允许只能看某些内容创建roles
创建用户，开发小于，绑定dev角色
yu
yu123123开发小于登录kibana
只能访问t1 index数据
4.角色访问index区别
END
官方站点：www.linuxprobe.com
Linux命令大全：www.linuxcool.com
刘遄老师QQ：5604215
Linux技术交流群：2636170
（新群，火热加群中……）
想要学习Linux系统的读者可以点击"阅读原文"按钮来了解书籍《Linux就该这么学》，同时也非常适合专业的运维人员阅读，成为辅助您工作的高价值工具书！
微信扫码关注该文公众号作者
戳这里提交新闻线索和高质量文章给我们。
来源: qq
点击查看作者最近其他文章