【es迁移】es端口非默认9200，使用logstash迁移脚本

4 年 ago

宋, 晓坤

3 minutes

问题版本：

ES升级或数据迁移时使用

问题现象/报错：

之前我们有开发提供的关于ES数据从5.6.7版本迁移到7.8.1版本的迁移脚本结合logstash使用，但端口是默认的9200，应该如何处理

排查思路/解决方案：

不要直接手动在脚本中修改9200端口，因为涉及到了其他变量的调用，所以只改查到的9200不可以。

在auditsys.ini文件中指定ES地址时，添加上端口，格式：ip：端口，如：192.168.1.66:9222

需要在原有脚本基础上替换dataMigrate.sh

替换脚本内容如下：

（无法以附件形式上传，可以百度云下载：链接：https://pan.baidu.com/s/1pXtc6u4nLENhfGVybuYFUw
提取码：cazl）

source ./auditsys.ini

sourcehost=$(echo $sourcehost)
inithost=$(echo $inithost)
initdate=$(echo $enddate)
startdate=$(echo $startdate)
enddate=$(echo $enddate)
initdate=${initdate//-0/-}

inittimes=$(date -d “$initdate” +%s)
starttimes=$(date -d “$startdate” +%s)

while [ $starttimes -le $inittimes ]
do

logstashfile=auditsys-$initdate.conf

cp auditsys.conf ./conf/$logstashfile
sed -i “s/_sourcehost_/$sourcehost/g” ./conf/$logstashfile
sed -i “s/_inithost_/$inithost/g” ./conf/$logstashfile
sed -i “s/_sourceindex_/metadata-$initdate/g” ./conf/$logstashfile

sourcecount=$(bash getSourceCount.sh http://$sourcehost/metadata-$initdate/_count)

imetadatacount=$(bash getInitCount.sh http://$inithost/meta-metadata-$initdate/_count)
if [[ $imetadatacount == “unknown error” ]];
then
imetadatacount=0
fi

iclickcount=$(bash getInitCount.sh http://$inithost/meta-click-$initdate/_count)
if [[ $iclickcount == “unknown error” ]];
then
iclickcount=0
fi

isessioncount=$(bash getInitCount.sh http://$inithost/meta-session-$initdate/_count)
if [[ $isessioncount == “unknown error” ]];
then
isessioncount=0
fi

iinfractcount=$(bash getInitCount.sh http://$inithost/meta-infract-$initdate/_count)
if [[ $iinfractcount == “unknown error” ]];
then
iinfractcount=0
fi

iefficiencycount=$(bash getInitCount.sh http://$inithost/meta-efficiency-$initdate/_count)
if [[ $iefficiencycount == “unknown error” ]];
then
iefficiencycount=0
fi

let “iinittotal=$imetadatacount+$iclickcount+$isessioncount+$iinfractcount+$iefficiencycount”

echo “sourcecount:$sourcecount”
echo “iinittotal:$iinittotal”
if [[ $sourcecount == “unknown error” ]];
then
sleep 1
echo “curl -s http://$sourcehost/metadata-$initdate/_count unknown error”
continue
elif [[ $sourcecount == “0” ]];
then
sourceResult=$(curl -s http://$sourcehost/metadata-$initdate/_count)
echo “sourceResult:$sourceResult”
sleep 1
echo “curl -s http://$sourcehost/metadata-$initdate/_count count is 0”
afterdate=$(date -d “$initdate -1 day ” +%Y-%m-%d)
afterdate=${afterdate//-0/-}
sed -i “s/enddate=$initdate/enddate=$afterdate/g” auditsys.ini
initdate=$afterdate
initdate=${initdate//-0/-}
inittimes=$(date -d “$initdate” +%s)
source ./auditsys.ini
continue
fi
echo “file:$logstashfile”
bash /usr/share/logstash/bin/logstash -f ./conf/$logstashfile

metadatacount=$(bash getInitCount.sh http://$inithost/meta-metadata-$initdate/_count)
if [[ $metadatacount == “unknown error” ]];
then
sleep 1
echo “curl -s http://$inithost/meta-metadata-$initdate/_count unknown error”
continue
fi

clickcount=$(bash getInitCount.sh http://$inithost/meta-click-$initdate/_count)
if [[ $clickcount == “unknown error” ]];
then
sleep 1
echo “curl -s http://$inithost/meta-click-$initdate/_count unknown error”
continue
fi

sessioncount=$(bash getInitCount.sh http://$inithost/meta-session-$initdate/_count)
if [[ $sessioncount == “unknown error” ]];
then
sleep 1
echo “curl -s http://$inithost/meta-session-$initdate/_count unknown error”
continue
fi

infractcount=$(bash getInitCount.sh http://$inithost/meta-infract-$initdate/_count)
if [[ $infractcount == “unknown error” ]];
then
sleep 1
echo “curl -s http://$sourcehost/meta-infract-$initdate/_count unknown error”
continue
fi

efficiencycount=$(bash getInitCount.sh http://$inithost/meta-efficiency-$initdate/_count)
if [[ $efficiencycount == “unknown error” ]];
then
sleep 1
echo “curl -s http://$inithost/meta-efficiency-$initdate/_count unknown error”
continue
fi

let “totalcount=$metadatacount+$sessioncount+$clickcount+$infractcount+$efficiencycount”
echo “totalcount:$totalcount”
let “totalcount=$totalcount-$iinittotal”
echo “synctotalcount:$totalcount”

if [[ “$totalcount” == “$sourcecount” ]];
then

echo “$initdate success,sourcecount:$sourcecount,initcount:$totalcount”
afterdate=$(date -d “$initdate -1 day ” +%Y-%m-%d)
afterdate=${afterdate//-0/-}
sed -i “s/enddate=$initdate/enddate=$afterdate/g” auditsys.ini
initdate=$afterdate
initdate=${initdate//-0/-}
inittimes=$(date -d “$initdate” +%s)
source ./auditsys.ini

else

echo “error totalcount:$totalcount sourcecount:$sourcecount”

done