Avoid offset lookup for newly created files
continuous-integration/drone Build is passing
Details
continuous-integration/drone Build is passing
Details
This commit is contained in:
parent
965b737548
commit
2bbcb1a51b
|
@ -254,7 +254,7 @@ async def uploader(coll, queue):
|
||||||
|
|
||||||
|
|
||||||
class LogFile(object):
|
class LogFile(object):
|
||||||
def __init__(self, coll, queue, path, namespace_name, pod_name, container_name, start=False):
|
def __init__(self, coll, queue, path, namespace_name, pod_name, container_name, start=False, lookup_offset=True):
|
||||||
self.offset = 0
|
self.offset = 0
|
||||||
self.path = path
|
self.path = path
|
||||||
self.buf = b""
|
self.buf = b""
|
||||||
|
@ -267,6 +267,7 @@ class LogFile(object):
|
||||||
self.coll = coll
|
self.coll = coll
|
||||||
self._state = None
|
self._state = None
|
||||||
self.state = "init"
|
self.state = "init"
|
||||||
|
self.lookup_offset = lookup_offset
|
||||||
if start:
|
if start:
|
||||||
self.start()
|
self.start()
|
||||||
|
|
||||||
|
@ -304,7 +305,7 @@ class LogFile(object):
|
||||||
"host.id": host_info["id"],
|
"host.id": host_info["id"],
|
||||||
"log.file.path": self.path
|
"log.file.path": self.path
|
||||||
}, sort=[("log.offset", -1)])
|
}, sort=[("log.offset", -1)])
|
||||||
histogram_database_operation_latency.labels("find-replay-offset").observe(time() - then)
|
histogram_database_operation_latency.labels("offset-lookup").observe(time() - then)
|
||||||
if last_record:
|
if last_record:
|
||||||
self.offset = last_record["log"]["offset"]
|
self.offset = last_record["log"]["offset"]
|
||||||
counter_skipped_bytes.inc(self.offset)
|
counter_skipped_bytes.inc(self.offset)
|
||||||
|
@ -434,7 +435,7 @@ class LogFile(object):
|
||||||
async def watcher(queue, coll):
|
async def watcher(queue, coll):
|
||||||
print("Starting watching")
|
print("Starting watching")
|
||||||
with Inotify() as inotify:
|
with Inotify() as inotify:
|
||||||
def add_file(path, finished=False, start=False):
|
def add_file(path, finished=False, start=False, lookup_offset=True):
|
||||||
if path in log_files:
|
if path in log_files:
|
||||||
log_files[path].finished = finished
|
log_files[path].finished = finished
|
||||||
return log_files[path]
|
return log_files[path]
|
||||||
|
@ -454,7 +455,8 @@ async def watcher(queue, coll):
|
||||||
return
|
return
|
||||||
if args.namespace and namespace_name != args.namespace:
|
if args.namespace and namespace_name != args.namespace:
|
||||||
return
|
return
|
||||||
lf = log_files[path] = LogFile(coll, queue, path, namespace_name, pod_name, container_name, start=start)
|
lf = log_files[path] = LogFile(coll, queue, path, namespace_name,
|
||||||
|
pod_name, container_name, start, lookup_offset)
|
||||||
lf.finished = finished
|
lf.finished = finished
|
||||||
inotify.add_watch(path, Mask.MODIFY | Mask.CLOSE_WRITE)
|
inotify.add_watch(path, Mask.MODIFY | Mask.CLOSE_WRITE)
|
||||||
return lf
|
return lf
|
||||||
|
@ -493,7 +495,7 @@ async def watcher(queue, coll):
|
||||||
# Events for /var/log/containers
|
# Events for /var/log/containers
|
||||||
if event.mask & Mask.CREATE:
|
if event.mask & Mask.CREATE:
|
||||||
counter_inotify_events.labels("create").inc()
|
counter_inotify_events.labels("create").inc()
|
||||||
add_file(os.path.realpath(event.path), start=True)
|
add_file(os.path.realpath(event.path), start=True, lookup_offset=False)
|
||||||
|
|
||||||
# Events for /var/log/pods
|
# Events for /var/log/pods
|
||||||
elif event.mask & Mask.CLOSE_WRITE:
|
elif event.mask & Mask.CLOSE_WRITE:
|
||||||
|
|
Loading…
Reference in New Issue