The Change Of Game Program No.2

发表于 更新于 分类于 阅读次数： Waline： 本文字数： 2.3k 阅读时长 ≈ 8 分钟
Game Part.2：关于remote机器如何使用bash脚本适配实现docker镜像信息同步到docker-swarm集群的整个过程

docker-swarm 部署流程：

  • 基于前一篇，每当在jenkins构建完之后，都会生成与对应项目分支相关的镜像（线上也如此配置，区别在于若线上环境则仅发布，开发/测试环境则发布与部署），但是线上的服务器是弹性伸缩式定时启动，同时为了实现更快地启动，暂时先存镜像到对应环境上的registry，在某个时刻再部署启动对应端口的service
1
2
3
4
5
6
7
8
9
10
11
# 脚本框架
CGManage
├── bin
│   ├── battleserver-docker.sh
│   ├── start_up.sh
│   └── shutdown.sh
├── config
│   ├── battleserver_config_docker
│   └── common_config
└── modules
    └── swarm_release
  • 需求分析
需求 配置文件及描述
config文件夹 battleserver_config_docker中进行定义统一的服务数量配置以及服务内相关参数的配置
swarm_release 基于config文件夹的内容进行具体服务配置的适配，并且把启动模版保存到config中，等待启动时进行某些必要的配置后进行相应的service启动
start_up.sh 弹性伸缩组的启动脚本模版
shutdown.sh 弹性伸缩组的退出脚本模版
  • 效果展示
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
FROM centos7:某镜像

ADD package/jdk-8u144-linux-x64.tar.gz /usr/local/jdk/

ENV JAVA_HOME=/usr/local/jdk/jdk1.8.0_144
ENV JRE_HOME=/usr/local/jdk/jdk1.8.0_144/jre
ENV PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin

ENV TZ=PRC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone

COPY/ADD 业务代码 # 区别在于是目录还是包

WORKDIR /工作目录

ENTRYPOINT ["/bin/bash","-c", "chmod +x {启动脚本} && exec {启动脚本} console {workspaceNum}"]
battleserver-docker.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
#/bin/bash
#逻辑程序区域
#获取
ARGS=$(getopt -a -o f:t:p:h -l deploy,name:,num:,ip:,env:,git: -- "$@")
#防止变量里有命令未能执行
eval set -- "${ARGS}"
while true
do
    case "$1" in
    --deploy)
            Action="deploy"
            ;;
    --name)
            ProjectName="$2"
            shift
            ;;
    --num)
            nodeCode="$2"
            shift
            ;;
    --ip)
            hostIp="$2"
            shift
            ;;
    --env)
            branchName="$2"
            Identifier="$2"
            shift
            ;;
    --git)
            gitName="$2"
            shift
            ;;
    --)
            shift
            break
            ;;
    esac
shift
done
#参数设定区域
cd $(dirname $0)
#加载配置文件
. /usr/local/CGManage/config/common_config
. /usr/local/CGManage/config/battleserver_config_docker
. /usr/local/CGManage/moudle/swarm_release

#常规变量
StartupParameters=.env
LaunchTemplate=docker-compose-swarm.yml
DeploymentDocumentation=docker_build.log
WorkspaceName="${BattleWorkspace}"

#遍历所有变量
for WORKSPACE in ${WorkspaceName}
do
	#声明字典
	declare -A $(echo ${WORKSPACE})
	#遍历所有Key
	for Key in $(eval echo '$'{!${WORKSPACE}Docker[@]})
	do
		#检查配置
		if [ ! "$(eval echo '$'{${WORKSPACE}Docker[$Key]})" ];then
			echo -e "\033[31mERROR:\033[0m ${ProjectName} ${WORKSPACE} 的 ${WORKSPACE}Docker[$Key] 配置出错,请添加配置或进行修改"
			exit 3
		fi
		#获取所有key到公共变量
		eval ${WORKSPACE}[$Key]=$(eval echo "'$'{${WORKSPACE}Docker[$Key]}")
	done
done
main_function
swarm_release
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
#!/bin/bash
#先备份到文件夹，然后进行解压、更新配置、停止程序、备份旧代码、新代码更新、启动程序

#发布并且启动项目
function main_function() {
	if [ "$Action" == "deploy" ];then
		CodePath="${DefaultDeployFileRoot}/${gitName}"
		release_code
	fi
}

#发布deploy
function release_code() {
	update_config
	backup_old_code
	release_new_code
	deploy_code
}

#更新线上配置
function update_config() {
    for nodeID in $(docker node ls | awk '{print $1}')
        do
            docker node inspect ${nodeID} | jq -r '.[] | .Spec.Labels' | grep "${MasterGitLabel}" | grep -q "${MasterGitValue}"
            if [ $? -eq 0 ];then
                hostIp=$(docker node inspect ${nodeID} --format "{{ .Status.Addr }}")
                break
            fi
        done
	for WORKSPACE in ${WorkspaceName}
	do
		echo -e "${WORKSPACE} :"
		ProjectsConfigPath=${BackupReleaseIdentifier}/${Time}/${ProjectName}${nodeCode}/${WORKSPACE}
		StartupParametersConfigPath=${ProjectsConfigPath}/${StartupParameters}
      	LaunchTemplateConfigPath=${ProjectsConfigPath}/${LaunchTemplate}
		if [ ! -d ${ProjectsConfigPath} ];then
			mkdir -p ${ProjectsConfigPath}
		fi
		if [ ! -f ${StartupParametersConfigPath} ];then
        	cp -f ${DefaultDeployFileRoot}/${gitName}/container/${StartupParameters} ${ProjectsConfigPath}/
        	dos2unix ${StartupParametersConfigPath}
		fi
		if [ ! -f ${LaunchTemplateConfigPath} ]; then
			cp -f ${DefaultDeployFileRoot}/${gitName}/container/${LaunchTemplate} ${ProjectsConfigPath}/
			dos2unix ${LaunchTemplateConfigPath}
		fi
		for Key in $(eval echo '$'{!${WORKSPACE}[@]})
		do
			# swarm.env
			# echo -e "\n\nswarm.env:"
			value="$(eval echo '$'{${WORKSPACE}[$Key]})"
			echo "${value}" | grep -q '@'
			if [[ $? -eq 0 ]];then
				echo "${value}" | grep '@' | grep -q '\\@'
				if [[ $? -ne 0 ]];then
					value=$(echo "${value}" | sed 's+\@+\\@+g')
				fi
			fi
			echo "${value}" | grep -q '&'
				if [[ $? -eq 0 ]];then
				echo "${value}" | grep '&' | grep -q '\\&'
				if [[ $? -ne 0 ]];then
					value=$(echo "${value}" | sed 's+\&+\\&+g')
				fi
			fi
			# swarm.env
			SedResult=$(sed -n 's@\(.*\)\('"${Key}"'\)\(.*\)\(=\)\(.*\)@\1\2\3\4'"${value}"'@p' ${StartupParametersConfigPath})
			if [ "${SedResult}" ];then
				sed -i 's@\(.*\)\('"${Key}"'\)\(.*\)\(=\)\(.*\)@\1\2\3\4'"${value}"'@; s@^M@@g' ${StartupParametersConfigPath}
				echo "1更改配置为: ${SedResult}"
				continue
			fi
			# docker-compose-swarm.yml
			SedResult=$(sed -n 's@\(.*\)\(\${'"${Key}"'}\)\(.*\)@\1'"${value}"'\3@p' ${LaunchTemplateConfigPath})
			if [ "${SedResult}" ];then
				sed -i 's@\(.*\)\(\${'"${Key}"'}\)\(.*\)@\1'"${value}"'\3@; s@^M@@g' ${LaunchTemplateConfigPath}
				echo "2更改配置为: ${SedResult}"
			fi
		done
		# 替换镜像
		IMAGE_NAME=$(cat "${DefaultDeployFileRoot}"/"${gitName}"/container/${DeploymentDocumentation} | grep "imageName" | awk -F "imageName@-@" '{for (i = 2; i <= NF; i++) printf "%s" , $i }' | sed 's/^[ \t]*//;s/[ \t]*$//')
		TAG=$(echo -n "${IMAGE_NAME}" | awk -F ":" '{print $2}')
		IMAGE_NAME="${hostIp}:${RegistryPort}/${ProjectName}:${TAG}"
		SedResult=$(sed -n 's@\(.*\)\(\${IMAGE_NAME}\)\(.*\)@\1'"${IMAGE_NAME}"'\3@p' ${StartupParametersConfigPath})
		if [ "${SedResult}" ];then
			sed -i 's@\(.*\)\(\${IMAGE_NAME}\)\(.*\)@\1'"${IMAGE_NAME}"'\3@; s@^M@@g' ${StartupParametersConfigPath}
			echo "6更改配置为: ${SedResult}"
		fi
		# 更新nodeCode
		SedResult=$(sed -n 's@\(.*\)\(\${nodeCode}\)\(.*\)@\1'"${nodeCode}"'\3@p' ${StartupParametersConfigPath})
		if [ "${SedResult}" ];then
			sed -i 's@\(.*\)\(\${nodeCode}\)\(.*\)@\1'"${nodeCode}"'\3@; s@^M@@g' ${StartupParametersConfigPath}
			echo "6更改配置为: ${SedResult}"
		fi
		# 更新hostIp
		SedResult=$(sed -n 's@\(.*\)\(\${nodeIp}\)\(.*\)@\1'"${hostIp}"'\3@p' ${StartupParametersConfigPath})
		if [ "${SedResult}" ];then
			sed -i 's@\(.*\)\(\${nodeIp}\)\(.*\)@\1'"${hostIp}"'\3@; s@^M@@g' ${StartupParametersConfigPath}
			echo "6更改配置为: ${SedResult}"
		fi
		# docker-compose-swarm.yml
		# echo -e "\n\ndocker-compose-swarm.yml:"
		while IFS='=' read -r key value; do
			# 跳过空行和注释
			echo "${value}" | grep -q '@'
			if [[ $? -eq 0 ]];then
			echo "${value}" | grep '@' | grep -q '\\@'
			if [[ $? -ne 0 ]];then
				value=$(echo "${value}" | sed 's+\@+\\@+g')
			fi
			fi
			echo "${value}" | grep -q '&'
			if [[ $? -eq 0 ]];then
			echo "${value}" | grep '&' | grep -q '\\&'
			if [[ $? -ne 0 ]];then
				value=$(echo "${value}" | sed 's+\&+\\&+g')
			fi
			fi
			[[ -z "$key" || "$key" =~ ^# ]] && continue
			while true;do
				SedResult=$(sed -n 's@\(.*\)\(\${'"${key}"'}\)\(.*\)@\1'"${value}"'\3@p' ${LaunchTemplateConfigPath})
				if [ "${SedResult}" ];then
					sed -i 's@\(.*\)\(\${'"${key}"'}\)\(.*\)@\1'"${value}"'\3@g; s@^M@@g' ${LaunchTemplateConfigPath}
					echo "3更改配置为: ${SedResult}"
				fi
				SedResult=$(sed -n 's@\(.*\)\(\${'"${key}"'}\)\(.*\)@\1'"${value}"'\3@p' ${LaunchTemplateConfigPath})
				if [ "${SedResult}" == "" ];then
					break
				fi
			done
		done < ${StartupParametersConfigPath}
		# 刷新/etc/hosts中的域名解析
		extra_hosts_location=$(grep -n 'extra_hosts:' ${LaunchTemplateConfigPath} | cut -d: -f1)
		extra_hosts_location=$((${extra_hosts_location}+1))
		sed -n ''"${extra_hosts_location}"'p' ${LaunchTemplateConfigPath} | grep -q "-"
		if [ $? -eq 0 ];then
			placeholders=$(sed -n ''"${extra_hosts_location}"'p' ${LaunchTemplateConfigPath} | awk -F "-" '{print $1}')
		else
			placeholders="      "
		fi
		extra_hosts_location=$((${extra_hosts_location}-1))
		# 基于/etc/hosts自动添加到extra_hots中
		while IFS= read -r line; do
			# 忽略空行和注释行
			if [[ "$line" =~ ^#.*$ ]] || [[ -z "$line" ]]; then
				continue
			fi
			# 解析IP地址和主机名
			programIp=$(echo "$line" | awk '{print $1}')
			if [[ "$programIp" =~ "127.0.0.1" ]] || [[ "$programIp" =~ "::1" ]];then
				continue
			fi
			programHosts=$(echo "$line" | awk '{for (i=2; i<=NF; i++) printf $i " "}')
			for programHost in ${programHosts[@]}
    		do
				echo "Adding $programHost -> $programIp to extra_hosts"
				sed -i "${extra_hosts_location} a\\${placeholders}- \"${programHost}:${programIp}\"" ${LaunchTemplateConfigPath}
			done
		done < "/etc/hosts"
		CODE_VERSION=$(echo -n "${IMAGE_NAME}" | awk -F ":" '{print $NF}')
		echo "CODE_VERSION=${CODE_VERSION}" >> ${StartupParametersConfigPath}
		echo -e "\n"
	done
}

#备份旧代码
function backup_old_code() {
	for WORKSPACE in ${WorkspaceName}
    do
        #先确认是否存在备份旧代码
        if [ ! -d ${ProjectRoot}/${Identifier}/${ProjectName}${nodeCode}/${WORKSPACE} ];then
                echo -e "Can't find the old code , \033[31;1m Skip backup \033[0m"
                return
        fi

        #进行备份操作
        mkdir ${BackupProjectRoot}/${Time}/${WORKSPACE} -p || echo -e "\033[31;1m Create backup projects dir fail \033[0m"
        mv ${ProjectRoot}/${Identifier}/${ProjectName}${nodeCode}/${WORKSPACE} ${BackupProjectRoot}/${Time}/ || {
                echo -e "Backup ${Identifier} ${ProjectName} old code  \033[31;1;5m False \033[0m"
                exit 2
        }
        #再次确认
        if [ -d ${BackupProjectRoot}/${Time}/${WORKSPACE} ];then
                echo -e "Backup ${Identifier} ${ProjectName} old code  \033[32;1m OK \033[0m"   
        fi
	done
}



#更新代码
function release_new_code() {
  [[ -d ${ProjectRoot}/${Identifier}/${ProjectName}${nodeCode} ]] || {
    mkdir -p ${ProjectRoot}/${Identifier}/${ProjectName}${nodeCode}
  }
  for WORKSPACE in ${WorkspaceName}
    do
      mv ${BackupReleaseIdentifier}/${Time}/${ProjectName}${nodeCode}/${WORKSPACE} ${ProjectRoot}/${Identifier}/${ProjectName}${nodeCode}/ || {
        echo -e "\033[31;1m Release New Code fail \033[0m"
        echo "ERROR : Can not move ${BackupReleaseIdentifier}/${Time}/${ProjectName}${nodeCode}/${WORKSPACE} to ${ProjectRoot}/${Identifier}/${ProjectName}${nodeCode}/"
        exit 2
      }
      if [ -d  ${ProjectRoot}/${Identifier}/${ProjectName}${nodeCode}/${WORKSPACE} ];then
        echo -e "Release new code \033[32;1m OK \033[0m "
      fi
    done
}

# 部署新的配置
function deploy_code() {
  # 先检查网络部分(因为最好规划好网段，所以统一提前配置)
  # docker swarm init --force-new-cluster --default-addr-pool 192.168.200.0/24
  # docker network create --driver overlay --attachable --subnet=192.168.201.0/24 monitoring
  # docker network create --driver overlay --attachable --subnet=192.168.202.0/24 ${ProjectName}  
    #  ResultCount=$(docker network ls | grep "${ProjectName}")
    #  if [[ "${ResultCount}" == "" ]];then
    #    docker network create --driver overlay "${ProjectName}"
    #  fi
    #  # 为了新环境直接启用，把monitoring的network也配置上
    #  ResultCount=$(docker network ls | grep "monitoring")
    #  if [[ "${ResultCount}" == "" ]];then
    #    docker network create --driver overlay --attachable monitoring
    #  fi
  # 暂不部署，先配置到config中，由启动的机子进行部署
  for WORKSPACE in ${WorkspaceName}
    do
      cd ${ProjectRoot}/${Identifier}/${ProjectName}${nodeCode}/${WORKSPACE}/
      docker stack deploy -c ${ProjectRoot}/${Identifier}/${ProjectName}${nodeCode}/${WORKSPACE}/docker-compose-swarm.yml --with-registry-auth "${ProjectName}${nodeCode}"
    done
}

Ps.目前弹性伸缩组中的服务器均已worker的方式加入集群，然后执行完再退出集群，整个过程的任何配置将以api的方式访问master来让master执行相关的操作

start_up.sh(worker)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
#!/bin/bash
# 应用参数
ProjectName="{应用类型}"
ServiceName="{workspaceNum}"
envName="{envName}"
gitName="{gitName}"
# 仅需要预先创建好/data/logs，并且保证目录是干净避免权限问题即可
if [ -d /data/logs ];then
   rm -rf /data/logs
fi
[ -d /data/logs ] || {
   mkdir -p /data/logs
}
# 用于dockerMsg的解析
rpm -qa | grep -q "jq"
[ $? -ne 0 ] || {
   yum install -y jq
}
rpm -qa | grep -qE "^curl"
[ $? -ne 0 ] || {
   yum install -y curl
}
# 以worker加入集群
nodeStatus=$(docker info --format '{{json .Swarm}}' | jq '.LocalNodeState' | sed 's/\"//g')
if [ "${nodeStatus}" == "inactive" ];then
  while true
   do
    docker swarm join --token {swarmToken} {swarmMaserIp}:2377
    if [ $? -eq 0 ];then
         break
    fi
    sleep 1
   done
fi
# 向master的api服务器发送一条关于自己信息的http请求（直接走内网）
hostIp=$(ip route | grep "default" | grep -oP '(?<=src )[0-9]{1,3}(\.[0-9]{1,3}){3}')
nodeCode=$(hostname | grep -oP '(?<=Server)\d+')
masterIp=$(docker info --format json | jq -r '.Swarm.RemoteManagers[0].Addr' | awk -F ":" '{print $1}')
while true
do
   # 建立tcp连接超时时间为60s，最长curl操作时间为180s
   httpCode=$(curl --connect-timeout 60 --max-time 180 -o /dev/null -w "%{http_code}" -X POST "http://${masterIp}:6666/create" -H "Content-Type: application/json" -d "{\"hostIp\":\"${hostIp}\", \"nodeCode\":\"${nodeCode}\", \"nodeHostname\":\"$(hostname)\", \"projectName\":\"${ProjectName}\", \"envName\":\"${envName}\", \"gitName\":\"${gitName}\"}")
   if [ ${httpCode} -ne 200 ];then
      sleep 2
      continue
    else
      break
   fi
done
while true
do
   docker ps | grep "${ProjectName}${nodeCode}" | grep "Up" | grep -q "healthy"
   if [ $? -eq 0 ];then
      # 进行一些创建成功的结果上报机制
      break
   fi
   sleep 2
done
exit 0
shutdown.sh(worker)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

#!/bin/bash
ProjectName="{应用类型}"
ServiceName="{workspaceNum}"
envName="{envName}"
gitName="{gitName}"
hostIp=$(ip route | grep "default" | grep -oP '(?<=src )[0-9]{1,3}(\.[0-9]{1,3}){3}')
nodeCode=$(hostname | grep -oP '(?<=Server)\d+')
masterIp=$(docker info --format json | jq -r '.Swarm.RemoteManagers[0].Addr' | awk -F ":" '{print $1}')
flag="true"
# 先检查本地是否容器服务在运行(如果启动出现问题,则可以直接退集群)
docker ps -a | grep -q "${ProjectName}${nodeCode}_${ServiceName}"
if [ $? -ne 0 ];then
  flag="false"
fi
if [ "${flag}" == "true" ];then
   while true
   do
      # 建立tcp连接超时时间为60s，最长curl操作时间为180s
      httpCode=$(curl --connect-timeout 60 --max-time 180 -o /dev/null -w "%{http_code}" -X POST "http://${masterIp}:6666/delete" -H "Content-Type: application/json" -d "{\"hostIp\":\"${hostIp}\", \"nodeCode\":\"${nodeCode}\", \"nodeHostname\":\"$(hostname)\", \"projectName\":\"${ProjectName}\", \"envName\":\"${envName}\", \"gitName\":\"${gitName}\"}")
      if [ ${httpCode} -eq 200 ];then
         break
      fi
      sleep 2
   done
fi
# 等待服务真正的清除完毕
if [ "${flag}" == "true" ];then
   while true
   do
      docker ps -a | grep -q "${ProjectName}${nodeCode}_${ServiceName}"
      if [ $? -ne 0 ];then
         break
      fi
      sleep 2
   done
fi
# 退出集群
nodeStatus=$(docker info --format '{{json .Swarm}}' | jq '.LocalNodeState' | sed 's/\"//g')
if [ "${nodeStatus}" == "active" ];then
   while true
      do
         docker swarm leave --force
         if [ $? -eq 0 ];then
            break
         fi
         sleep 1
  done
fi
if [ -d /data/logs ];then
   rm -rf /data/logs
fi
for dockerId in $(docker ps | grep "cadvisor" | awk '{print $1}')
do
  docker stop ${dockerId}
done
docker system prune -a -f
exit 0