Logging handler to send logs to your OpenSearch cluster with bulk SSL. Forked from https://github.com/logzio/logzio-python-handler
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
216 lines
7.9 KiB
216 lines
7.9 KiB
# This class is responsible for handling all asynchronous Logz.io's
|
|
# communication
|
|
import sys
|
|
import json
|
|
|
|
from time import sleep
|
|
from datetime import datetime
|
|
from threading import Thread, enumerate
|
|
|
|
import requests
|
|
from datetime import datetime
|
|
|
|
import opensearchpy
|
|
|
|
import logging
|
|
|
|
opensearch_logger = logging.getLogger('opensearch')
|
|
opensearch_logger.propagate = False
|
|
opensearch_logger.handlers.clear()
|
|
opensearch_logger.setLevel(logging.WARNING)
|
|
|
|
|
|
from .logger import get_logger
|
|
from .logger import get_stdout_logger
|
|
|
|
if sys.version[0] == '2':
|
|
import Queue as queue
|
|
else:
|
|
import queue as queue
|
|
|
|
MAX_BULK_SIZE_IN_BYTES = 1 * 1024 * 1024 # 1 MB
|
|
|
|
|
|
def backup_logs(logs, logger):
|
|
timestamp = datetime.now().strftime('%d%m%Y-%H%M%S')
|
|
logger.info(
|
|
'Backing up your logs to logzio-failures-%s.txt', timestamp)
|
|
with open('logzio-failures-{}.txt'.format(timestamp), 'a') as f:
|
|
f.writelines('\n'.join(logs))
|
|
|
|
|
|
class LogzioSender:
|
|
def __init__(self,
|
|
host, port, username, password,
|
|
# token, url='https://listener.logz.io:8071',
|
|
logs_drain_timeout=5,
|
|
debug=False,
|
|
backup_logs=True,
|
|
network_timeout=10.0,
|
|
number_of_retries=4,
|
|
retry_timeout=2):
|
|
# self.token = token
|
|
# self.url = '{}/?token={}'.format(url, token)
|
|
|
|
self.client = opensearchpy.OpenSearch(
|
|
hosts = [{'host': host, 'port': port}],
|
|
http_compress = True, # enables gzip compression for request bodies
|
|
http_auth = (username, password),
|
|
use_ssl = True,
|
|
verify_certs = True,
|
|
ssl_assert_hostname = False,
|
|
ssl_show_warn = False,
|
|
)
|
|
self.logs_drain_timeout = logs_drain_timeout
|
|
self.stdout_logger = get_stdout_logger(debug)
|
|
self.backup_logs = backup_logs
|
|
self.network_timeout = network_timeout
|
|
self.requests_session = requests.Session()
|
|
self.number_of_retries = number_of_retries
|
|
self.retry_timeout = retry_timeout
|
|
|
|
# Function to see if the main thread is alive
|
|
self.is_main_thread_active = lambda: any(
|
|
(i.name == 'MainThread') and i.is_alive() for i in enumerate())
|
|
|
|
# Create a queue to hold logs
|
|
self.queue = queue.Queue()
|
|
self._initialize_sending_thread()
|
|
|
|
def __del__(self):
|
|
del self.stdout_logger
|
|
del self.backup_logs
|
|
del self.queue
|
|
|
|
def _initialize_sending_thread(self):
|
|
self.sending_thread = Thread(target=self._drain_queue)
|
|
self.sending_thread.daemon = False
|
|
self.sending_thread.name = 'logzio-sending-thread'
|
|
self.sending_thread.start()
|
|
|
|
def append(self, logs_message):
|
|
if not self.sending_thread.is_alive():
|
|
self._initialize_sending_thread()
|
|
|
|
# Queue lib is thread safe, no issue here
|
|
# self.queue.put(json.dumps(logs_message))
|
|
self.queue.put(logs_message)
|
|
|
|
def flush(self):
|
|
self._flush_queue()
|
|
|
|
def _drain_queue(self):
|
|
last_try = False
|
|
|
|
while not last_try:
|
|
# If main is exited, we should run one last time and try to remove
|
|
# all logs
|
|
if not self.is_main_thread_active():
|
|
self.stdout_logger.debug(
|
|
'Identified quit of main thread, sending logs one '
|
|
'last time')
|
|
last_try = True
|
|
|
|
try:
|
|
self._flush_queue()
|
|
except Exception as e:
|
|
self.stdout_logger.debug(
|
|
'Unexpected exception while draining queue to Logz.io, '
|
|
'swallowing. Exception: %s', e)
|
|
|
|
if not last_try:
|
|
sleep(self.logs_drain_timeout)
|
|
|
|
def _flush_queue(self):
|
|
# Sending logs until queue is empty
|
|
while not self.queue.empty():
|
|
logs_list = self._get_messages_up_to_max_allowed_size()
|
|
self.stdout_logger.debug(
|
|
'Starting to drain %s logs to Logz.io', len(logs_list))
|
|
|
|
# Not configurable from the outside
|
|
sleep_between_retries = self.retry_timeout
|
|
self.number_of_retries = self.number_of_retries
|
|
|
|
should_backup_to_disk = True
|
|
headers = {"Content-type": "text/plain"}
|
|
|
|
for current_try in range(self.number_of_retries):
|
|
should_retry = False
|
|
try:
|
|
index_name = f"backendlog-{datetime.utcnow():%Y-%m}"
|
|
index_body = {'settings': {'index': {'number_of_shards': 1, 'number_of_replicas': 0}}}
|
|
self.client.indices.create(index_name, body=index_body, ignore=400)
|
|
respose = opensearchpy.helpers.bulk(
|
|
self.client,
|
|
[{'_index': index_name, **entry} for entry in logs_list],
|
|
max_retries=3
|
|
)
|
|
|
|
# response = self.requests_session.post(
|
|
# self.url, headers=headers, data='\n'.join(logs_list),
|
|
# timeout=self.network_timeout)
|
|
# if response.status_code != 200:
|
|
# if response.status_code == 400:
|
|
# self.stdout_logger.info(
|
|
# 'Got 400 code from Logz.io. This means that '
|
|
# 'some of your logs are too big, or badly '
|
|
# 'formatted. response: %s', response.text)
|
|
# should_backup_to_disk = False
|
|
# break
|
|
|
|
# if response.status_code == 401:
|
|
# self.stdout_logger.info(
|
|
# 'You are not authorized with Logz.io! Token '
|
|
# 'OK? dropping logs...')
|
|
# should_backup_to_disk = False
|
|
# break
|
|
# else:
|
|
# self.stdout_logger.info(
|
|
# 'Got %s while sending logs to Logz.io, '
|
|
# 'Try (%s/%s). Response: %s',
|
|
# response.status_code,
|
|
# current_try + 1,
|
|
# self.number_of_retries,
|
|
# response.text)
|
|
# should_retry = True
|
|
# else:
|
|
self.stdout_logger.debug(
|
|
'Successfully sent bulk of %s logs to '
|
|
'Logz.io!', len(logs_list))
|
|
should_backup_to_disk = False
|
|
break
|
|
except Exception as e:
|
|
self.stdout_logger.warning(
|
|
'Got exception while sending logs to Logz.io, '
|
|
'Try (%s/%s). Message: %s',
|
|
current_try + 1, self.number_of_retries, e)
|
|
should_retry = True
|
|
|
|
if should_retry:
|
|
sleep(sleep_between_retries)
|
|
|
|
if should_backup_to_disk and self.backup_logs:
|
|
# Write to file
|
|
self.stdout_logger.error(
|
|
'Could not send logs to Logz.io after %s tries, '
|
|
'backing up to local file system', self.number_of_retries)
|
|
backup_logs(logs_list, self.stdout_logger)
|
|
|
|
del logs_list
|
|
|
|
def _get_messages_up_to_max_allowed_size(self):
|
|
logs_list = []
|
|
current_size = 0
|
|
while not self.queue.empty():
|
|
current_log = self.queue.get()
|
|
try:
|
|
current_size += sys.getsizeof(current_log)
|
|
except TypeError:
|
|
# pypy do not support sys.getsizeof
|
|
current_size += len(current_log) * 4
|
|
|
|
logs_list.append(current_log)
|
|
if current_size >= MAX_BULK_SIZE_IN_BYTES:
|
|
break
|
|
return logs_list
|