From e7f9585b6c222ad4bcf018725f992679f9ae40ec Mon Sep 17 00:00:00 2001 From: tamir-michaeli Date: Mon, 10 Jan 2022 11:18:15 +0200 Subject: [PATCH] bug fixes --- logzio/handler.py | 14 +++++++------ logzio/logger.py | 13 +++++++++++-- logzio/sender.py | 39 +++++++++++++++++++++---------------- requirements.txt | 2 +- setup.py | 7 ++++--- tests/test_logzioHandler.py | 19 ++++++++++-------- tests/test_logzioSender.py | 12 ++++++++---- 7 files changed, 65 insertions(+), 41 deletions(-) diff --git a/logzio/handler.py b/logzio/handler.py index d412747..9fca847 100644 --- a/logzio/handler.py +++ b/logzio/handler.py @@ -18,7 +18,9 @@ class LogzioHandler(logging.Handler): url="https://listener.logz.io:8071", debug=False, backup_logs=True, - network_timeout=10.0): + network_timeout=10.0, + retries_no=4, + retry_timeout=2): if not token: raise LogzioException('Logz.io Token must be provided') @@ -31,7 +33,9 @@ class LogzioHandler(logging.Handler): logs_drain_timeout=logs_drain_timeout, debug=debug, backup_logs=backup_logs, - network_timeout=network_timeout) + network_timeout=network_timeout, + number_of_retries=retries_no, + retry_timeout=retry_timeout) logging.Handler.__init__(self) def __del__(self): @@ -94,10 +98,8 @@ class LogzioHandler(logging.Handler): if message.exc_info: return_json['exception'] = self.format_exception(message.exc_info) - # We want to ignore default logging formatting on exceptions - # As we handle those differently directly into exception field - message.exc_info = None - message.exc_text = None + # # We want to ignore default logging formatting on exceptions + # # As we handle those differently directly into exception field formatted_message = self.format(message) if isinstance(formatted_message, dict): diff --git a/logzio/logger.py b/logzio/logger.py index 0dcacca..f120485 100644 --- a/logzio/logger.py +++ b/logzio/logger.py @@ -3,7 +3,16 @@ import logging def get_logger(debug): - logger = logging.getLogger(__name__) + return __get_logger(debug, __name__) + + +def get_stdout_logger(debug): + return __get_logger(debug, __name__ + '_stdout', logging.StreamHandler(sys.stdout)) + + +def __get_logger(debug, name, handler=None): + logger = logging.getLogger(name) logger.setLevel(logging.DEBUG if debug else logging.INFO) - logger.addHandler(logging.StreamHandler(sys.stdout)) + if handler: + logger.addHandler(handler) return logger diff --git a/logzio/sender.py b/logzio/sender.py index d158dab..ca73b73 100644 --- a/logzio/sender.py +++ b/logzio/sender.py @@ -10,13 +10,13 @@ from threading import Thread, enumerate import requests from .logger import get_logger +from .logger import get_stdout_logger if sys.version[0] == '2': import Queue as queue else: import queue as queue - MAX_BULK_SIZE_IN_BYTES = 1 * 1024 * 1024 # 1 MB @@ -34,14 +34,19 @@ class LogzioSender: logs_drain_timeout=5, debug=False, backup_logs=True, - network_timeout=10.0): + network_timeout=10.0, + number_of_retries=4, + retry_timeout=2): self.token = token self.url = '{}/?token={}'.format(url, token) self.logs_drain_timeout = logs_drain_timeout self.logger = get_logger(debug) + self.stdout_logger = get_stdout_logger(debug) self.backup_logs = backup_logs self.network_timeout = network_timeout self.requests_session = requests.Session() + self.number_of_retries = number_of_retries + self.retry_timeout = retry_timeout # Function to see if the main thread is alive self.is_main_thread_active = lambda: any( @@ -53,6 +58,7 @@ class LogzioSender: def __del__(self): del self.logger + del self.stdout_logger del self.backup_logs del self.queue @@ -79,7 +85,7 @@ class LogzioSender: # If main is exited, we should run one last time and try to remove # all logs if not self.is_main_thread_active(): - self.logger.debug( + self.stdout_logger.debug( 'Identified quit of main thread, sending logs one ' 'last time') last_try = True @@ -98,17 +104,17 @@ class LogzioSender: # Sending logs until queue is empty while not self.queue.empty(): logs_list = self._get_messages_up_to_max_allowed_size() - self.logger.debug( + self.stdout_logger.debug( 'Starting to drain %s logs to Logz.io', len(logs_list)) # Not configurable from the outside - sleep_between_retries = 2 - number_of_retries = 4 + sleep_between_retries = self.retry_timeout + self.number_of_retries = self.number_of_retries should_backup_to_disk = True headers = {"Content-type": "text/plain"} - for current_try in range(number_of_retries): + for current_try in range(self.number_of_retries): should_retry = False try: response = self.requests_session.post( @@ -116,7 +122,7 @@ class LogzioSender: timeout=self.network_timeout) if response.status_code != 200: if response.status_code == 400: - self.logger.info( + self.stdout_logger.info( 'Got 400 code from Logz.io. This means that ' 'some of your logs are too big, or badly ' 'formatted. response: %s', response.text) @@ -124,42 +130,41 @@ class LogzioSender: break if response.status_code == 401: - self.logger.info( + self.stdout_logger.info( 'You are not authorized with Logz.io! Token ' 'OK? dropping logs...') should_backup_to_disk = False break else: - self.logger.info( + self.stdout_logger.info( 'Got %s while sending logs to Logz.io, ' 'Try (%s/%s). Response: %s', response.status_code, current_try + 1, - number_of_retries, + self.number_of_retries, response.text) should_retry = True else: - self.logger.debug( + self.stdout_logger.debug( 'Successfully sent bulk of %s logs to ' 'Logz.io!', len(logs_list)) should_backup_to_disk = False break except Exception as e: - self.logger.warning( + self.stdout_logger.warning( 'Got exception while sending logs to Logz.io, ' 'Try (%s/%s). Message: %s', - current_try + 1, number_of_retries, e) + current_try + 1, self.number_of_retries, e) should_retry = True if should_retry: sleep(sleep_between_retries) - sleep_between_retries *= 2 if should_backup_to_disk and self.backup_logs: # Write to file - self.logger.error( + self.stdout_logger.error( 'Could not send logs to Logz.io after %s tries, ' - 'backing up to local file system', number_of_retries) + 'backing up to local file system', self.number_of_retries) backup_logs(logs_list, self.logger) del logs_list diff --git a/requirements.txt b/requirements.txt index b7ec5aa..dff3f70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -requests>=2.23.0 \ No newline at end of file +requests>=2.27.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 8e5cd3f..4e70646 100644 --- a/setup.py +++ b/setup.py @@ -7,12 +7,13 @@ setup( description="Logging handler to send logs to your Logz.io account with bulk SSL", keywords="logging handler logz.io bulk https", author="roiravhon", - author_email="roi@logz.io", + maintainer="tamir-michaeli", + mail="tamir.michaeli@logz.io", url="https://github.com/logzio/logzio-python-handler/", license="Apache License 2", packages=find_packages(), install_requires=[ - "requests>=2.23.0" + "requests>=2.27.0" ], test_requires=[ "future" @@ -21,6 +22,6 @@ setup( classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', - 'Programming Language :: Python :: 2.7' + 'Programming Language :: Python :: 3.9' ] ) diff --git a/tests/test_logzioHandler.py b/tests/test_logzioHandler.py index b65ff27..60ec3ec 100644 --- a/tests/test_logzioHandler.py +++ b/tests/test_logzioHandler.py @@ -15,7 +15,8 @@ class TestLogzioHandler(TestCase): def test_json(self): formatter = logging.Formatter( - '{ "appname":"%(name)s", "functionName":"%(funcName)s", \"lineNo":"%(lineno)d", "severity":"%(levelname)s", "message":"%(message)s"}') + '{ "appname":"%(name)s", "functionName":"%(funcName)s", \"lineNo":"%(lineno)d", "severity":"%(' + 'levelname)s", "message":"%(message)s"}') self.handler.setFormatter(formatter) record = logging.LogRecord( @@ -74,7 +75,7 @@ class TestLogzioHandler(TestCase): 'message': 'this is a test: moo.', 'path_name': 'handler_test.py', 'type': 'python' - } + } ) def test_extra_formatting(self): @@ -105,7 +106,7 @@ class TestLogzioHandler(TestCase): 'path_name': 'handler_test.py', 'type': 'python', 'extra_key': 'extra_value' - } + } ) def test_format_string_message(self): @@ -133,7 +134,7 @@ class TestLogzioHandler(TestCase): 'message': 'this is a test: moo.', 'path_name': 'handler_test.py', 'type': 'python' - } + } ) def test_exc(self): @@ -147,7 +148,7 @@ class TestLogzioHandler(TestCase): level=0, pathname='handler_test.py', lineno=10, - msg="this is a test: moo.", + msg='exception test:', args=[], exc_info=exc_info, func='test_json' @@ -158,6 +159,7 @@ class TestLogzioHandler(TestCase): formatted_message["exception"] = formatted_message["exception"].replace(os.path.abspath(__file__), "") formatted_message["exception"] = re.sub(r", line \d+", "", formatted_message["exception"]) + formatted_message["message"] = formatted_message["message"].replace(os.path.abspath(__file__), "") self.assertDictEqual( { @@ -165,11 +167,12 @@ class TestLogzioHandler(TestCase): 'line_number': 10, 'log_level': 'NOTSET', 'logger': 'my-logger', - 'message': 'this is a test: moo.', - 'exception': 'Traceback (most recent call last):\n\n File "", in test_exc\n raise ValueError("oops.")\n\nValueError: oops.\n', + 'message': f'exception test:\nTraceback (most recent call last):\n File "", line 142, in test_exc\n raise ' + 'ValueError("oops.")\nValueError: oops.', + 'exception': 'Traceback (most recent call last):\n\n File "", in test_exc\n raise ValueError(' + '"oops.")\n\nValueError: oops.\n', 'path_name': 'handler_test.py', 'type': 'python' }, formatted_message ) - diff --git a/tests/test_logzioSender.py b/tests/test_logzioSender.py index 78dba04..da91084 100644 --- a/tests/test_logzioSender.py +++ b/tests/test_logzioSender.py @@ -24,6 +24,8 @@ class TestLogzioSender(TestCase): self.logzio_listener.clear_logs_buffer() self.logzio_listener.clear_server_error() self.logs_drain_timeout = 1 + self.retries_no = 4 + self.retry_timeout = 2 logging_configuration = { "version": 1, @@ -42,7 +44,9 @@ class TestLogzioSender(TestCase): 'logzio_type': "type", 'logs_drain_timeout': self.logs_drain_timeout, 'url': "http://" + self.logzio_listener.get_host() + ":" + str(self.logzio_listener.get_port()), - 'debug': True + 'debug': True, + 'retries_no': self.retries_no, + 'retry_timeout': self.retry_timeout } }, "loggers": { @@ -88,7 +92,7 @@ class TestLogzioSender(TestCase): self.logzio_listener.clear_server_error() - time.sleep(self.logs_drain_timeout * 2 * 4) # Longer, because of the retry + time.sleep(self.logs_drain_timeout * self.retry_timeout * self.retries_no) # Longer, because of the retry self.assertTrue(self.logzio_listener.find_log(log_message)) @@ -100,7 +104,7 @@ class TestLogzioSender(TestCase): # Make sure no file is present self.assertEqual(len(_find("logzio-failures-*.txt", ".")), 0) - time.sleep(2 * 2 * 2 * 2 * 2) # All of the retries + time.sleep(self.retries_no*self.retry_timeout*2*2) # All of the retries failure_files = _find("logzio-failures-*.txt", ".") self.assertEqual(len(failure_files), 1) @@ -118,7 +122,7 @@ class TestLogzioSender(TestCase): # Make sure no file is present self.assertEqual(len(_find("logzio-failures-*.txt", ".")), 0) - time.sleep(2 * 2 * 2 * 2 * 2) # All of the retries + time.sleep(self.retries_no*self.retry_timeout) # All of the retries # Make sure no file was created self.assertEqual(len(_find("logzio-failures-*.txt", ".")), 0)