Skip to content

Commit 46a24e5

Browse files
deepcompute-agentdeepcompute-agent
authored
Merge pull request #89 from deep-compute/update_haproxyhandler_logic
update haproxy handler code of logagg ref #87
2 parents 7863bc1 + bab2f74 commit 46a24e5

1 file changed

Lines changed: 76 additions & 49 deletions

File tree

‎logagg/formatters.py‎

Lines changed: 76 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -16,75 +16,102 @@ def docker_file_log_driver(line):
1616
)
1717
return dict(timestamp=log.get('timestamp'), data=log, type='log')
1818

19+
HAPROXY_HEADERS = re.compile(r'{(.*)} ')
1920
def haproxy(line):
2021
#TODO Handle all message formats
2122
'''
2223
>>> import pprint
23-
>>> input_line1 = 'Apr 24 00:00:02 node haproxy[12298]: 1.1.1.1:48660 [24/Apr/2019:00:00:02.358] pre-staging~ pre-staging_doc/pre-staging_active 261/0/2/8/271 200 2406 - - ---- 4/4/0/1/0 0/0 {AAAAAA:AAAAA_AAAAA:AAAAA_AAAAA_AAAAA:300A||| user@mail.net:sdasdasdasdsdasAHDivsjd=|user@mail.net|2018} "GET /doc/api/get?call=apple HTTP/1.1"'
24+
>>> input_line1 = 'Apr 24 00:00:02 node haproxy[12298]: 1.1.1.1:48660 [24/Apr/2019:00:00:02.358] pre-staging~ pre-staging_doc/pre-staging_active 261/0/2/8/271 200 2406 - - ---- 4/4/0/1/0 0/0 {AAAAAA:AAAAA_AAAAA:AAAAA_AAAAA_AAAAA:300A|||HMAC user@mail.net:sdasdasdasdsdasAHDivsjd=|user@mail.net|2018} "GET /doc/api/get?call=apple HTTP/1.1"'
2425
>>> output_line1 = haproxy(input_line1)
2526
>>> pprint.pprint(output_line1)
2627
{'data': {'Tc': 2.0,
27-
'Tq': 261.0,
28-
'Tr': 8.0,
29-
'Tw': 0.0,
30-
'_api': '/doc/api/get?call=apple',
31-
'_headers': ['AAAAAA:AAAAA_AAAAA:AAAAA_AAAAA_AAAAA:300A||| user@mail.net:sdasdasdasdsdasAHDivsjd=|user@mail.net|2018'],
32-
'actconn': 4,
33-
'backend': 'pre-staging_doc/pre-staging_active',
34-
'backend_queue': 0,
35-
'beconn': 1,
36-
'bytes_read': 2406.0,
37-
'client_port': '48660',
38-
'client_server': '1.1.1.1',
39-
'feconn': 4,
40-
'front_end': 'pre-staging~',
41-
'haproxy_server': 'node',
42-
'method': 'GET',
43-
'resp_time': 271.0,
44-
'retries': 0,
45-
'srv_conn': 0,
46-
'srv_queue': 0,
47-
'status': '200',
48-
'timestamp': '2019-04-24T00:00:02.358000'},
28+
'Tq': 261.0,
29+
'Tr': 8.0,
30+
'Tw': 0.0,
31+
'_headers': ['AAAAAA:AAAAA_AAAAA:AAAAA_AAAAA_AAAAA:300A',
32+
'',
33+
'',
34+
'HMAC user@mail.net:sdasdasdasdsdasAHDivsjd=',
35+
'user@mail.net',
36+
'2018'],
37+
'_url_params': {'call': 'apple'},
38+
'actconn': 4,
39+
'backend': 'pre-staging_doc/pre-staging_active',
40+
'backend_queue': 0,
41+
'beconn': 0,
42+
'bytes_read': 2406,
43+
'client_port': 48660,
44+
'client_server': '1.1.1.1',
45+
'feconn': 4,
46+
'front_end': 'pre-staging~',
47+
'haproxy_server': 'node',
48+
'http_version': 'HTTP/1.1',
49+
'method': 'GET',
50+
'resp_time': 271.0,
51+
'retries': 0,
52+
'srv_conn': 1,
53+
'srv_queue': 0,
54+
'status': '200',
55+
'timestamp': '2019-04-24T00:00:02.358000',
56+
'url_path': '/doc/api/get'},
4957
'event': 'haproxy_event',
5058
'timestamp': '2019-04-24T00:00:02.358000',
5159
'type': 'metric'}
5260
'''
5361

54-
_line = line.strip().split()
62+
_headers = HAPROXY_HEADERS.findall(line)
63+
if _headers:
64+
_headers = _headers[0].strip().split('|')
65+
else:
66+
_headers = []
67+
68+
_line = HAPROXY_HEADERS.sub('', line).strip().split()
69+
70+
_, _, _, haproxy_server, _, client_server_info, timestamp, front_end, \
71+
backend, resp_timings, status, bytes_read, \
72+
_, _, _, conn_stats, inqueue_stats, method, url, http_version = _line
73+
74+
keys = [
75+
'_headers', 'haproxy_server', 'client_server',
76+
'client_port', 'timestamp', 'front_end', 'backend',
77+
'Tq', 'Tw', 'Tc', 'Tr', 'resp_time', 'status',
78+
'bytes_read', 'retries', 'actconn', 'feconn',
79+
'beconn', 'srv_conn', 'srv_queue', 'backend_queue',
80+
'method', 'url_path', '_url_params', 'http_version'
81+
]
82+
83+
haproxy_server = haproxy_server.strip()
84+
85+
client_server, client_port = client_server_info.split(':')
86+
client_server = client_server.strip()
87+
client_port = int(client_port.strip())
88+
89+
_timestamp = re.findall(r'\[(.*?)\]', timestamp)[0]
90+
timestamp = datetime.datetime.strptime(_timestamp, '%d/%b/%Y:%H:%M:%S.%f').isoformat()
5591

56-
log = {}
57-
log['client_server'] = _line[5].split(':')[0].strip()
58-
log['client_port'] = _line[5].split(':')[1].strip()
92+
front_end = front_end.strip()
93+
backend = backend.strip()
5994

60-
_timestamp = re.findall(r'\[(.*?)\]', _line[6])[0]
61-
log['timestamp'] = datetime.datetime.strptime(_timestamp, '%d/%b/%Y:%H:%M:%S.%f').isoformat()
95+
Tq, Tw, Tc, Tr, resp_time = [float(rt.strip()) for rt in resp_timings.split('/')]
6296

63-
log['front_end'] = _line[7].strip()
64-
log['backend'] = _line[8].strip()
97+
status = status.strip()
98+
bytes_read = int(bytes_read.strip())
6599

66-
log['Tq'] = float(_line[9].split('/')[0].strip())
67-
log['Tw'] = float(_line[9].split('/')[1].strip())
68-
log['Tc'] = float(_line[9].split('/')[2].strip())
69-
log['Tr'] = float(_line[9].split('/')[3].strip())
70-
log['resp_time'] = float(_line[9].split('/')[-1].strip())
71-
log['status'] = _line[10].strip()
72-
log['bytes_read'] = float(_line[11].strip())
100+
actconn, feconn, beconn, srv_conn, retries = [int(cs.strip()) for cs in conn_stats.split('/')]
101+
srv_queue, backend_queue = [int(inqstat.strip()) for inqstat in inqueue_stats.split('/')]
102+
method = method.strip('"').strip()
73103

74-
log['_headers'] = re.findall(r'{(.*)}', line)
75-
log['haproxy_server'] = _line[3].strip()
104+
_url = url.strip().split('?')
105+
url_path, url_params = [u.strip() for u in _url]
76106

77-
log['method'] = _line[-3].strip('"').strip()
78-
log['_api'] = _line[-2].strip()
107+
_url_params = {}
108+
for params in url_params.split('&'):
109+
key, val = params.strip().split('=')
110+
_url_params[key.strip()] = val.strip()
79111

80-
log['retries'] = int(_line[15].split('/')[-1].strip())
81-
log['actconn'] = int(_line[15].split('/')[0].strip())
82-
log['feconn'] = int(_line[15].split('/')[1].strip())
83-
log['beconn'] = int(_line[15].split('/')[-2].strip())
84-
log['srv_conn'] = int(_line[15].split('/')[-3].strip())
112+
http_version = http_version.strip('"').strip()
85113

86-
log['srv_queue'] = int(_line[16].split('/')[0].strip())
87-
log['backend_queue'] = int(_line[16].split('/')[1].strip())
114+
log = dict(zip(keys, [eval(k) for k in keys]))
88115

89116
return dict(
90117
data=log,

0 commit comments

Comments
 (0)