The Change Of Game Program No.3

发表于 更新于 分类于 阅读次数： Waline： 本文字数： 1.4k 阅读时长 ≈ 5 分钟
Game Part.3：关于如何使用go脚本实现弹性伸缩组的机器基于http-api的方式创建属于该节点的service的整个过程

api调用过程(实现最简单化的api-server的功能)：

1、client端向http://masterIp:6666/[create、delete、health]发送post请求（请求体包含了需要过滤的条件）
2、server端接收到请求后，根据需求进行对应的操作（create创建对应节点所对应端口的service，delete删除对应节点所对应端口的service，health为了docker-compose中的health实现容器自维护）
3、server端把执行结果返回给client端

HttpApi.go：
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
// export GOPROXY=https://goproxy.cn
// go mod init HttpApi
// go get github.com/docker/docker/api/types && go get github.com/docker/docker/client
// go mod tidy
// CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o HttpApi
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/client"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

// Params 定义了将从 POST 请求中解析的参数
type Params struct {
	HostIp       string `json:"hostIp"`
	EnvName      string `json:"envName"`
	NodeCode     string `json:"nodeCode"`
	GitName      string `json:"gitName"`
	NodeHostname string `json:"nodeHostname"`
	ProjectName  string `json:"projectName"`
}

func main() {
	http.HandleFunc("/create", handlerCreate) // 创建Service
	http.HandleFunc("/delete", handlerDelete) // 删除Service
	http.HandleFunc("/health", handlerStatus) // 给healthcheck使用
	//log.Println("Listening on http://localhost:6666/")
	// log.Fatalf()--Printf log.Fatal()--Print log.Fatalln()--Println 这些都会自带os.Exit(1)，所以可以利用这个搭配swarm的重启策略来实现服务的重启
	log.Fatal(http.ListenAndServe(":6666", nil))
}

func handlerStatus(w http.ResponseWriter, r *http.Request) {
	// 仅接收post格式的http请求
	if r.Method != "GET" {
		http.Error(w, "http api just accept on post, other will deny", http.StatusBadRequest)
		return
	}
	// 发送响应
	w.WriteHeader(http.StatusOK)
	w.Write([]byte("status ok"))
}

func handlerCreate(w http.ResponseWriter, r *http.Request) {
	// 仅接收post格式的http请求
	if r.Method != "POST" {
		http.Error(w, "http api just accept on post, other will deny", http.StatusBadRequest)
		return
	}
	// 解析 JSON 格式的请求体
	var params Params
	err := json.NewDecoder(r.Body).Decode(&params)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	fmt.Printf("params: %#v\n", params)
	// 获取Swarm集群的Master节点IP
	// masterIP, err := getMasterIP()
	// if err != nil {
	// 	http.Error(w, "master must have something error", http.StatusBadRequest)
	// 	return
	// }
	dockerHost := "unix:///var/run/docker.sock"
	ctx := context.Background()
	// 创建 Docker 客户端，自动从环境变量配置
	cli, err := client.NewClientWithOpts(client.WithHost(dockerHost), client.WithAPIVersionNegotiation())
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	// 在执行完当前函数的所有内容才会退出docker-client连接
	defer cli.Close()
	// 获取标签keyName 格式为export NodeLabelsRule=battleserverNum
	nodeLabelRule, err := getNodeLabelRule("NodeLabelsRule")
	if err != nil {
		http.Error(w, "Failed to get node label rule", http.StatusBadRequest)
		return
	}
	// 修改node label
	_, err = changeNodeByHostname(ctx, cli, nodeLabelRule, &params)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	// 部署相应的service
	err = createService(&params)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	// 发送响应
	w.WriteHeader(http.StatusOK)
	w.Write([]byte("create finished"))
}

func handlerDelete(w http.ResponseWriter, r *http.Request) {
	// 仅接收post格式的http请求
	if r.Method != "POST" {
		http.Error(w, "http api just accept on post, other will deny", http.StatusBadRequest)
		return
	}
	// 解析 JSON 格式的请求体
	var params Params
	err := json.NewDecoder(r.Body).Decode(&params)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	fmt.Printf("params: %#v\n", params)
	// 获取Swarm集群的Master节点IP
	// masterIP, err := getMasterIP()
	// if err != nil {
	// 	http.Error(w, "master must have something error", http.StatusBadRequest)
	// 	return
	// }
	dockerHost := "unix:///var/run/docker.sock"
	ctx := context.Background()
	// 创建 Docker 客户端，自动从环境变量配置
	cli, err := client.NewClientWithOpts(client.WithHost(dockerHost), client.WithAPIVersionNegotiation())
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	// 在执行完当前函数的所有内容才会退出docker-client连接
	defer cli.Close()
	// 删除某个指定的服务
	serviceName := params.ProgramName + params.NodeCode + "_" + params.ServiceName
	err = cli.ServiceRemove(ctx, serviceName)
	if err != nil {
		http.Error(w, "Failed to remove "+serviceName, http.StatusBadRequest)
		return
	}
	// 发送响应
	w.WriteHeader(http.StatusOK)
	w.Write([]byte("delete finished"))
}

// 获取环境变量
func getNodeLabelRule(labelName string) (string, error) {
	envVars := os.Environ()
	for _, env := range envVars {
		pair := strings.SplitN(env, "=", 2)
		if len(pair) == 2 && pair[0] == labelName {
			return pair[1], nil
		}
	}
	return "", fmt.Errorf("Rule for node %s not found", labelName)
}

// 解析masterip，但容器化之后无法访问集群，所以无法使用
func getMasterIP() (string, error) {
	var out bytes.Buffer
	cmd := exec.Command("docker", "info", "-f", "{{.Swarm.NodeAddr}}")
	cmd.Stdout = &out
	if err := cmd.Run(); err != nil {
		return "", err
	}
	return strings.TrimSpace(out.String()), nil
}

// changeNodeByHostname 根据节点的hostname修改节点的label
func changeNodeByHostname(ctx context.Context, dockerCli *client.Client, nodeLabelRule string, params *Params) (string, error) {
	nodes, err := dockerCli.NodeList(ctx, types.NodeListOptions{})
	if err != nil {
		return "", err
	}
	for _, node := range nodes {
		if node.Description.Hostname == params.NodeHostname {
			node.Spec.Annotations.Labels[nodeLabelRule] = params.NodeCode
			err := dockerCli.NodeUpdate(ctx, node.ID, node.Version, node.Spec)
			if err != nil {
				return "", err
			}
		}
	}
	return "", nil
}

// 部署特定的service
func createService(params *Params) error {
	cmd := exec.Command("/bin/bash", "/usr/local/CGManage/bin/battleserver-docker.sh", "--deploy", "--name", params.ProjectName, "--num", params.NodeCode, "--ip", params.HostIp, "--env", params.EnvName, "--git", params.GitName)
	_, err := cmd.CombinedOutput()
	if err != nil {
		return err
	}
	if exitError, ok := err.(*exec.ExitError); ok {
		// 获取退出状态码
		exitCode := exitError.ExitCode()
		// 根据状态码进行相应的操作
		if exitCode != 0 {
			return fmt.Errorf("createService run error")
		}
	}
	return nil
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
FROM alpine:latest

# deploy&&publish script
ADD CGManage.tar.gz /usr/local/

COPY HttpApi /opt/

RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories && \
    apk add --no-cache docker bash bash-completion jq curl && \
    chmod +x /opt/cktHttp

WORKDIR /opt

CMD ["/opt/HttpApi"]
docker-compose-httpapi.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
services:
  api:
    image: "{httpapi镜像}"
    user: "0:0"
    ports:
      - "6666:6666"
    logging:
      driver: "json-file"
      options:
        max-size: "1G"
        max-file: "3"
    environment:
      - NodeLabelsRule=battleserverSeq
      - MasterGitLabel=git-cache
      - MasterGitValue=true
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /usr/local/CGManage/config/battleserver_config_docker:/usr/local/CGManage/config/battleserver_config_docker
      - /etc/hosts:/etc/hosts:ro
      - /data:/data
    networks:
      - monitoring
    healthcheck:
      test: [ "CMD", "curl", "-f", "http://127.0.0.1:6666/health" ]
      interval: 10s
      timeout: 10s
      retries: 3
    cap_add:
      - SYS_PTRACE
    deploy:
      mode: replicated # 部署方式，deployment还是damonset
      replicas: 1 # 副本数，damonset不可用
      endpoint_mode: vip # 提供给外部分别与端口对应的vip
      placement:
        constraints:
          - node.role == manager
          - # docker node update --label-add git-cache=true masterID
          - "node.labels.git-cache == true"
      resources:
        limits:
          memory: 500M
          cpus: '1.0'
      restart_policy:
        condition: any
        delay: 0s
        max_attempts: 99999999
        window: 5s

networks:
  # docker network create --driver overlay --attachable monitoring
  monitoring:
    driver: overlay
    external: true
    attachable: true
# docker stack deploy -c docker-compose-swarm.yml --with-registry-auth http
registry api 使用：
1
2
3
4
5
6
7
# registry api：
# 列出所有的仓库：curl http://192.168.20.104:5000/v2/_catalog
# 先docker tag xxx:xxx 192.168.20.104:5000/xxx:xxx，docker push 192.168.20.104:5000/xxx:xxx, 节点就可以pull
# 获取某个仓库的tags-list：curl http://192.168.20.104:5000/v2/xxx/tags/list
# 获取某个仓库的某个tag的digest：curl --header "Accept: application/vnd.docker.distribution.manifest.v2+json" -I -s -X GET http://192.168.20.104:5000/v2/xxx/manifests/tag | grep Docker-Content-Digest | awk '{print $2}' | tr -d '\r'
# 删除某个镜像的元数据，标识已被删除，但还未能从磁盘释放空间：curl --header "Accept: application/vnd.docker.distribution.manifest.v2+json" -X DELETE  http://192.168.20.104:5000/v2/xxx/manifests/${imageSha}
# 触发仓库的gc回收空间，释放磁盘空间，但不会刷新registry的缓存记录，对于同一tag重复提交会有一定影响使用：docker exec -i registry_containerdID /bin/sh -c "/bin/registry garbage-collect /etc/docker/registry/config.yml"