k-space
/
logmower-shipper-prototype
9
0
Fork 0
Code

Fix UTF-8 handling
continuous-integration/drone Build is passing Details

This commit is contained in:
Lauri Võsandi 2022-11-06 22:55:38 +02:00
parent 5b5b76c534
commit 434f3bba39
1 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
log_shipper.py
View File

@ -169,7 +169,7 @@ class LogFile(object):
self.path = path self.path = path
self.tail = 0 self.tail = 0
self.more_content = asyncio.Event() self.more_content = asyncio.Event()
self.fh = open(path) self.fh = open(path, "rb")
self.queue = queue self.queue = queue
self.namespace_name = namespace_name self.namespace_name = namespace_name
self.pod_name = pod_name self.pod_name = pod_name
@ -221,16 +221,23 @@ class LogFile(object):
assert self.head < self.tail assert self.head < self.tail
self.fh.seek(self.head) self.fh.seek(self.head)
buf = self.fh.readline() buf = self.fh.readline()
try: try:
if len(buf) < 45: reason = "unicode-encoding"
line = buf.decode("utf-8")
if len(line) < 45:
reason = "line-short"
raise ValueError() raise ValueError()
if not buf[-1] == "\n": if not line[-1] == "\n":
reason = "no-newline %s" % repr((buf))
raise ValueError() raise ValueError()
if not re.match("^(.+) (stdout|stderr)( (.))? (.*)$", buf[:-1]): if not re.match("^(.+) (stdout|stderr)( (.))? (.*)$", line[:-1]):
reason = "no-regex-match"
raise ValueError() raise ValueError()
event_created = datetime.strptime(buf[:23], "%Y-%m-%dT%H:%M:%S.%f") reason = "invalid-timestamp"
event_created = datetime.strptime(line[:23], "%Y-%m-%dT%H:%M:%S.%f")
except ValueError: except ValueError:
print("Failed to parse file %s at offset %d" % (self.path, self.head)) print("Failed to parse file %s at offset %d, reason: %s" % (self.path, self.head, reason))
break break
histogram_line_size.observe(len(buf)) histogram_line_size.observe(len(buf))
@ -239,9 +246,9 @@ class LogFile(object):
if record_size < args.max_record_size: if record_size < args.max_record_size:
# TODO: Support Docker runtime on EKS # TODO: Support Docker runtime on EKS
message += buf[45:-1] message += line[45:-1]
state = buf[43] state = line[43]
if state == "P": if state == "P":
# This is partial message # This is partial message
continue continue
@ -257,7 +264,7 @@ class LogFile(object):
# TODO: Log portion of the message # TODO: Log portion of the message
continue continue
stream = buf[36:42].strip() stream = line[36:42].strip()
if message.startswith("{\""): if message.startswith("{\""):
# TODO: Follow Filebeat hints # TODO: Follow Filebeat hints
try: try: