First Automation scripts

发表于 更新于 分类于 阅读次数： Waline： 本文字数： 1.2k 阅读时长 ≈ 4 分钟
Needs Part.1：使用ansible剧本和python-api的方式实现自动分析与收集某些类型的日志

api调用过程：

1、client端向http://vlog.anile.online:2814发送post请求（请求体包含了需要过滤的条件）
2、server端接收到请求后，根据需求进行对应的操作（如果系统的python版本是3.9+，则使用ansible-runner；如果系统的python版本是3.9-，则生成ansible-playbook，然后再通过bash脚本执行这个ansible-playbook）
3、server端把执行结果返回给client端

  • 结果展示：

The result

Script show：
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# workspace.yml
- gather_facts: 'no'
  hosts: socket-server
  remote_user: gamec
  tasks:
  - name: get each WorkspaceNum
    register: command_output
    shell: cat /usr/local/CGManage/pjc3c/config/mainserver_config | grep 'Workspace'
      | grep 'workspace' | awk -F '"' '{print $2}'
  - debug:
      var: command_output.stdout_lines
    name: Debug output

# {host}-site.yml
- gather_facts: 'no'
  hosts: socket-server1
  remote_user: gamec
  tasks:
  - name: gzip all log file
    shell: gzip /data/logs/pjc3c/mainserver/{{ item }}/behavior/behavior_*
    with_items: '{{ workspaces }}'
  - name: do something prepare
    shell: mkdir -p /data/logs/pjc3c/mainserver/tmp/;rm -rf /data/logs/pjc3c/mainserver/tmp/20*
  - ignore_errors: 'yes'
    loop: '{{ workspaces | product(dates) | list }}'
    name: operational definition
    shell: mkdir -p /data/logs/pjc3c/mainserver/tmp/{{ item[1] }};gunzip -cd /data/logs/pjc3c/mainserver/{{
      item[0] }}/behavior/behavior_{{ item[1] }}.log.gz | grep -E "3313301" > /data/logs/pjc3c/mainserver/tmp/{{
      item[1] }}/{{ item[0] }}.log
  - name: merge result
    shell: cd /data/logs/pjc3c/mainserver/tmp/;rm -f /data/logs/pjc3c/mainserver/tmp/behavior.tar.gz;tar
      -czf /data/logs/pjc3c/mainserver/tmp/behavior.tar.gz $(ls | grep "^20")
  vars:
    dates:
    - '2024-10-18'
    - '2024-10-19'
    - '2024-10-20'
    workspaces:
    - workspace1
    - workspace2
    - workspace3
    - workspace4
    - workspace99

基于目前环境中python的版本为3.6+为主，后续若更换为3.9+，则补充相应的ansible-runner方式

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
# pip3 install --upgrade -i https://mirrors.aliyun.com/pypi/simple pip
# pip install -i https://mirrors.aliyun.com/pypi/simple ansible
# pip install -i https://mirrors.aliyun.com/pypi/simple ansible-runner # python 3.9 以上才可以使用，很坑
import json
import os
import re
from http.server import BaseHTTPRequestHandler, HTTPServer
import urllib.parse
import sys
import yaml
from datetime import datetime, timedelta
import subprocess

class DoAnsible(object):

    def __init__(self):
        self.dirname = os.path.dirname(os.path.abspath(__file__))
        # 获取服务目录的地址
        self.cgmanage_path = "/usr/local/CGManage"

    def getLogType(self):
        self.logType = []
        with open(os.path.join(self.dirname, 'log-type.txt'), 'r') as file:
            for line in file.readlines():
                line = line.strip()
                if line == "":
                    continue
                self.logType.append(line)

    def getWorkspaceNum(self,projEnv, program):
        file_path = os.path.join(self.cgmanage_path, projEnv,"config",program+"_config")
        env = projEnv[4:]
        command = f"cat {file_path} | grep 'Workspace' | grep 'workspace' | awk -F '\"' '{{print $2}}'"
        playbook = {
            'hosts': 'socket-server',
            'remote_user': 'game'+env,
            'gather_facts': 'no',
            'tasks': [
                {
                    'name': 'get each WorkspaceNum',
                    'shell': command,
                    'register': 'command_output'
                },
                {
                    'name': 'Debug output',
                    'debug': {
                    'var': 'command_output.stdout_lines'
                    }
                }
            ]
        }
        playbook_file = os.path.join(self.dirname, 'workspace.yml')
        with open(playbook_file, 'w') as f:
            yaml.dump([playbook], f, default_flow_style=False)
        # Only python 3.7+ support capture_output=True and text=True
        result = subprocess.run(['ansible-playbook', playbook_file], check=True, stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE, timeout=300)
        if result.returncode == 0:
            output = result.stdout.decode('utf-8')
            # 用于匹配Ansible输出的正则表达式
            pattern = re.compile(r'ok: \[(.*?)\] => {\s+"command_output\.stdout_lines": \[\n\s+\"(.*?)"\n\s+\]\s+}')
            # 解析任务和输出
            return pattern.findall(output)
        else:
            return [("",""),]

    def checkDate(self,fromDate,toDate):
        if fromDate[:4] > toDate[:4]:
            return False
        elif fromDate[:4] == toDate[:4]:
            if int(fromDate[4:6]) > int(toDate[4:6]):
                return False
        return True

class RequestHandler(BaseHTTPRequestHandler):

    def do_POST(self):
        # 刷新支持的log-type类型
        self.doAnsible.getLogType()
        # 对URI进行解析
        parsed_path = urllib.parse.urlparse(self.path)
        # 获取路径
        paths = parsed_path.path[1:].split("/")
        if len(paths) == 1:
            method = paths[0]
            # 基于3.10才有match-case模块，所以只能继续使用if
            if method in self.doAnsible.logType:
                # 读取请求体的长度
                content_length = int(self.headers['Content-Length'])
                # 读取请求体中的数据
                post_data = self.rfile.read(content_length)
                # 解析JSON数据
                try:
                    data = json.loads(post_data.decode('utf-8'))
                    if data["date"].find("-") == -1:
                        # 仅对一天的日志进行操作
                        need_dates = [datetime.strptime(data["date"], '%Y%m%d').strftime('%Y-%m-%d'),]
                    else:
                        # 对多天的日志进行操作
                        fromDate = data["date"].split("-")[0]
                        toDate = data["date"].split("-")[1]
                        if not self.doAnsible.checkDate(fromDate,toDate):
                            self.send_error(404, "Date Configure Has something Wrong")
                            return
                        start_date = datetime.strptime(fromDate, '%Y%m%d')
                        end_date = datetime.strptime(toDate, '%Y%m%d')
                        current_date = start_date
                        need_dates = []
                        while current_date <= end_date:
                            need_dates.append(current_date.strftime('%Y-%m-%d'))
                            current_date += timedelta(days=1)  # 增加一天
                    # 获取目前某个特定的节点上分别有哪些workspace正在运行
                    host_workspace = self.doAnsible.getWorkspaceNum(self.projEnv, data["program"])
                    response_content = ""
                    for host , workspace_nums in host_workspace:
                        if host == "":
                            continue
                        env = self.projEnv[4:]
                        workspace_nums = workspace_nums.split(" ")
                        operational_method = 'mkdir -p /data/logs/%s/%s/tmp/{{ item[1] }};gunzip -cd /data/logs/%s/%s/{{ item[0] }}/%s/%s_{{ item[1] }}.log.gz | grep -E \"%s\" > /data/logs/%s/%s/tmp/{{ item[1] }}/{{ item[0] }}.log'%(self.projEnv,data["program"],self.projEnv,data["program"],method,method,data["rule"],self.projEnv,data["program"])
                        merge_method = 'cd /data/logs/%s/%s/tmp/;rm -f /data/logs/%s/%s/tmp/%s.tar.gz;tar -czf /data/logs/%s/%s/tmp/%s.tar.gz $(ls | grep \"^20\")'%(self.projEnv,data["program"],self.projEnv,data["program"],method,self.projEnv,data["program"],method)
                        playbook = {
                            'hosts': host,
                            'remote_user': 'game'+env,
                            'gather_facts': 'no',
                            'vars': {
                                'dates': need_dates,
                                'workspaces': workspace_nums
                            },
                            'tasks': [
                                {
                                    'name': 'gzip all log file',
                                    'shell': 'gzip /data/logs/%s/%s/{{ item }}/%s/%s_*'%(self.projEnv,data["program"],method,method),
                                    'with_items': '{{ workspaces }}'
                                },
                                {
                                    'name': 'do something prepare',
                                    'shell': 'mkdir -p /data/logs/%s/%s/tmp/;rm -rf /data/logs/%s/%s/tmp/20*'%(self.projEnv,data["program"],self.projEnv,data["program"])
                                },
                                {
                                    'name': 'operational definition',
                                    'shell': operational_method,
                                    'loop': '{{ workspaces | product(dates) | list }}',
                                    'ignore_errors': 'yes'
                                },
                                {
                                    'name': 'merge result',
                                    'shell': merge_method
                                }
                            ]
                        }
                        playbook_file = os.path.join(self.doAnsible.dirname, f'{host}-site.yml')
                        with open(playbook_file, 'w') as f:
                            yaml.dump([playbook], f, default_flow_style=False)
                        result = subprocess.run(['ansible-playbook', playbook_file], check=True, stdout=subprocess.PIPE,
                                                    stderr=subprocess.PIPE,timeout=300)
                        if result.returncode == 0:
                            workspace_num = host[13:]
                            response_content += f'done Successfully!!! Please come to vlog-snsjz{env}.xxx.com/{data["program"]}/{data["program"]}{workspace_num}/logs/tmp/{method}.tar.gz' + "\n"
                    if response_content != "":
                        self.send_response(200)
                        self.send_header('Content-type', 'application/json; charset=utf-8')
                        self.end_headers()
                        self.wfile.write(response_content.encode('utf-8'))
                        return
                    else:
                        self.send_error(404, "done Error!!!Please contact the System Administrator to resolv this problem...")
                        return
                except json.JSONDecodeError:
                    self.send_error(404,"Received data is not in JSON format")
                    return
            else:
                self.send_error(404, "Log Type is not supported")
                return
        else:
            self.send_error(404, "URL Params Has something Wrong")
            return

    def do_HEAD(self):
        # 为了monit而准备的 HEAD 请求
        self.send_response(200)
        self.send_header('Content-type', 'text/html')
        self.end_headers()

def run(projEnv, server_class=HTTPServer, handler_class=RequestHandler, port=2814):
    doAnsible = DoAnsible()
    server_address = ('', port)
    httpd = server_class(server_address, handler_class)
    setattr(handler_class, "doAnsible", doAnsible)
    setattr(handler_class, "projEnv", projEnv)
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        print('Good Bye Everyone! Good everning Everyone!')
        sys.exit(0)

if __name__ == '__main__':
    projEnv = sys.argv[1]
    # noinspection PyTypeChecker
    run(projEnv, port=2814)

Self-maintenance：
1
2
3
4
5
yum install -y epel-release
yum install -y monit
check host monitWeb with address 127.0.0.1
    if failed port 2814 type tcp then exec "/data/home/test_code/web-monit.sh pjc3c" as uid "gamec" and gid "game"