Add bulk insertion error counters
continuous-integration/drone Build is passing
Details
continuous-integration/drone Build is passing
Details
This commit is contained in:
parent
1e82d2af51
commit
3cf2ea9e28
|
@ -97,6 +97,10 @@ counter_insertion_errors = Counter(
|
||||||
"logmower_insertion_error_count",
|
"logmower_insertion_error_count",
|
||||||
"Exceptions caught during insertion of single event",
|
"Exceptions caught during insertion of single event",
|
||||||
["exception"])
|
["exception"])
|
||||||
|
counter_bulk_insertion_errors = Counter(
|
||||||
|
"logmower_bulk_insertion_error_count",
|
||||||
|
"Exceptions caught during bulk insertions",
|
||||||
|
["exception"])
|
||||||
counter_bulk_insertions = Counter(
|
counter_bulk_insertions = Counter(
|
||||||
"logmower_bulk_insertion_count",
|
"logmower_bulk_insertion_count",
|
||||||
"Count of bulk insertions to database",
|
"Count of bulk insertions to database",
|
||||||
|
@ -191,9 +195,10 @@ async def uploader(coll, queue):
|
||||||
o = await asyncio.wait_for(queue.get(), timeout=0.1)
|
o = await asyncio.wait_for(queue.get(), timeout=0.1)
|
||||||
except asyncio.exceptions.TimeoutError:
|
except asyncio.exceptions.TimeoutError:
|
||||||
break
|
break
|
||||||
gauge_queue_entries.set(queue.qsize())
|
else:
|
||||||
o["event"]["ingested"] = datetime.utcnow()
|
gauge_queue_entries.set(queue.qsize())
|
||||||
messages.append(o)
|
o["event"]["ingested"] = datetime.utcnow()
|
||||||
|
messages.append(o)
|
||||||
if not messages:
|
if not messages:
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
|
@ -202,10 +207,13 @@ async def uploader(coll, queue):
|
||||||
await coll.insert_many(messages)
|
await coll.insert_many(messages)
|
||||||
histogram_database_operation_latency.labels("insert-many").observe(time() - then)
|
histogram_database_operation_latency.labels("insert-many").observe(time() - then)
|
||||||
except pymongo.errors.ServerSelectionTimeoutError:
|
except pymongo.errors.ServerSelectionTimeoutError:
|
||||||
|
counter_bulk_insertions.labels("timed-out").inc()
|
||||||
continue
|
continue
|
||||||
except pymongo.errors.BulkWriteError as e:
|
except pymongo.errors.BulkWriteError as e:
|
||||||
print("Bulk insertion failed:", e)
|
counter_bulk_insertions.labels("retried-as-singles").inc()
|
||||||
counter_bulk_insertions.labels("failed").inc()
|
j = "%s.%s" % (e.__class__.__module__, e.__class__.__name__)
|
||||||
|
counter_bulk_insertion_errors.labels(j).inc()
|
||||||
|
print("Bulk insert failed: %s" % j)
|
||||||
for o in messages:
|
for o in messages:
|
||||||
o.pop("_id", None)
|
o.pop("_id", None)
|
||||||
o["event"]["ingested"] = datetime.utcnow()
|
o["event"]["ingested"] = datetime.utcnow()
|
||||||
|
|
Loading…
Reference in New Issue