Merge pull request #66 from logzio/fix-issues-42-46-60-61

bug fixes

logzio/handler.py

@@ -18,7 +18,9 @@ class LogzioHandler(logging.Handler):
                  url="https://listener.logz.io:8071",
                  debug=False,
                  backup_logs=True,
-                 network_timeout=10.0):
+                 network_timeout=10.0,
+                 retries_no=4,
+                 retry_timeout=2):
 
         if not token:
             raise LogzioException('Logz.io Token must be provided')
@@ -31,7 +33,9 @@ class LogzioHandler(logging.Handler):
             logs_drain_timeout=logs_drain_timeout,
             debug=debug,
             backup_logs=backup_logs,
-            network_timeout=network_timeout)
+            network_timeout=network_timeout,
+            number_of_retries=retries_no,
+            retry_timeout=retry_timeout)
         logging.Handler.__init__(self)
 
     def __del__(self):
@@ -94,10 +98,8 @@ class LogzioHandler(logging.Handler):
         if message.exc_info:
             return_json['exception'] = self.format_exception(message.exc_info)
 
-            # We want to ignore default logging formatting on exceptions
-            # As we handle those differently directly into exception field
-            message.exc_info = None
-            message.exc_text = None
+            # # We want to ignore default logging formatting on exceptions
+            # # As we handle those differently directly into exception field
 
         formatted_message = self.format(message)
         if isinstance(formatted_message, dict):

logzio/logger.py

@@ -3,7 +3,16 @@ import logging
 
 
 def get_logger(debug):
-    logger = logging.getLogger(__name__)
+    return __get_logger(debug, __name__)
+
+
+def get_stdout_logger(debug):
+    return __get_logger(debug, __name__ + '_stdout', logging.StreamHandler(sys.stdout))
+
+
+def __get_logger(debug, name, handler=None):
+    logger = logging.getLogger(name)
     logger.setLevel(logging.DEBUG if debug else logging.INFO)
-    logger.addHandler(logging.StreamHandler(sys.stdout))
+    if handler:
+        logger.addHandler(handler)
     return logger

logzio/sender.py

@@ -10,13 +10,13 @@ from threading import Thread, enumerate
 import requests
 
 from .logger import get_logger
+from .logger import get_stdout_logger
 
 if sys.version[0] == '2':
     import Queue as queue
 else:
     import queue as queue
 
-
 MAX_BULK_SIZE_IN_BYTES = 1 * 1024 * 1024  # 1 MB
 
 
@@ -34,14 +34,19 @@ class LogzioSender:
                  logs_drain_timeout=5,
                  debug=False,
                  backup_logs=True,
-                 network_timeout=10.0):
+                 network_timeout=10.0,
+                 number_of_retries=4,
+                 retry_timeout=2):
         self.token = token
         self.url = '{}/?token={}'.format(url, token)
         self.logs_drain_timeout = logs_drain_timeout
         self.logger = get_logger(debug)
+        self.stdout_logger = get_stdout_logger(debug)
         self.backup_logs = backup_logs
         self.network_timeout = network_timeout
         self.requests_session = requests.Session()
+        self.number_of_retries = number_of_retries
+        self.retry_timeout = retry_timeout
 
         # Function to see if the main thread is alive
         self.is_main_thread_active = lambda: any(
@@ -53,6 +58,7 @@ class LogzioSender:
 
     def __del__(self):
         del self.logger
+        del self.stdout_logger
         del self.backup_logs
         del self.queue
 
@@ -79,7 +85,7 @@ class LogzioSender:
             # If main is exited, we should run one last time and try to remove
             # all logs
             if not self.is_main_thread_active():
-                self.logger.debug(
+                self.stdout_logger.debug(
                     'Identified quit of main thread, sending logs one '
                     'last time')
                 last_try = True
@@ -98,17 +104,17 @@ class LogzioSender:
         # Sending logs until queue is empty
         while not self.queue.empty():
             logs_list = self._get_messages_up_to_max_allowed_size()
-            self.logger.debug(
+            self.stdout_logger.debug(
                 'Starting to drain %s logs to Logz.io', len(logs_list))
 
             # Not configurable from the outside
-            sleep_between_retries = 2
-            number_of_retries = 4
+            sleep_between_retries = self.retry_timeout
+            self.number_of_retries = self.number_of_retries
 
             should_backup_to_disk = True
             headers = {"Content-type": "text/plain"}
 
-            for current_try in range(number_of_retries):
+            for current_try in range(self.number_of_retries):
                 should_retry = False
                 try:
                     response = self.requests_session.post(
@@ -116,7 +122,7 @@ class LogzioSender:
                         timeout=self.network_timeout)
                     if response.status_code != 200:
                         if response.status_code == 400:
-                            self.logger.info(
+                            self.stdout_logger.info(
                                 'Got 400 code from Logz.io. This means that '
                                 'some of your logs are too big, or badly '
                                 'formatted. response: %s', response.text)
@@ -124,42 +130,41 @@ class LogzioSender:
                             break
 
                         if response.status_code == 401:
-                            self.logger.info(
+                            self.stdout_logger.info(
                                 'You are not authorized with Logz.io! Token '
                                 'OK? dropping logs...')
                             should_backup_to_disk = False
                             break
                         else:
-                            self.logger.info(
+                            self.stdout_logger.info(
                                 'Got %s while sending logs to Logz.io, '
                                 'Try (%s/%s). Response: %s',
                                 response.status_code,
                                 current_try + 1,
-                                number_of_retries,
+                                self.number_of_retries,
                                 response.text)
                             should_retry = True
                     else:
-                        self.logger.debug(
+                        self.stdout_logger.debug(
                             'Successfully sent bulk of %s logs to '
                             'Logz.io!', len(logs_list))
                         should_backup_to_disk = False
                         break
                 except Exception as e:
-                    self.logger.warning(
+                    self.stdout_logger.warning(
                         'Got exception while sending logs to Logz.io, '
                         'Try (%s/%s). Message: %s',
-                        current_try + 1, number_of_retries, e)
+                        current_try + 1, self.number_of_retries, e)
                     should_retry = True
 
                 if should_retry:
                     sleep(sleep_between_retries)
-                    sleep_between_retries *= 2
 
             if should_backup_to_disk and self.backup_logs:
                 # Write to file
-                self.logger.error(
+                self.stdout_logger.error(
                     'Could not send logs to Logz.io after %s tries, '
-                    'backing up to local file system', number_of_retries)
+                    'backing up to local file system', self.number_of_retries)
                 backup_logs(logs_list, self.logger)
 
             del logs_list

requirements.txt

@@ -1 +1 @@
-requests>=2.23.0
+requests>=2.27.0

setup.py

@@ -7,12 +7,13 @@ setup(
     description="Logging handler to send logs to your Logz.io account with bulk SSL",
     keywords="logging handler logz.io bulk https",
     author="roiravhon",
-    author_email="roi@logz.io",
+    maintainer="tamir-michaeli",
+    mail="tamir.michaeli@logz.io",
     url="https://github.com/logzio/logzio-python-handler/",
     license="Apache License 2",
     packages=find_packages(),
     install_requires=[
-        "requests>=2.23.0"
+        "requests>=2.27.0"
     ],
     test_requires=[
         "future"
@@ -21,6 +22,6 @@ setup(
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',
-        'Programming Language :: Python :: 2.7'
+        'Programming Language :: Python :: 3.9'
     ]
 )

tests/test_logzioHandler.py

@@ -15,7 +15,8 @@ class TestLogzioHandler(TestCase):
 
     def test_json(self):
         formatter = logging.Formatter(
-            '{ "appname":"%(name)s", "functionName":"%(funcName)s", \"lineNo":"%(lineno)d", "severity":"%(levelname)s", "message":"%(message)s"}')
+            '{ "appname":"%(name)s", "functionName":"%(funcName)s", \"lineNo":"%(lineno)d", "severity":"%('
+            'levelname)s", "message":"%(message)s"}')
         self.handler.setFormatter(formatter)
 
         record = logging.LogRecord(
@@ -74,7 +75,7 @@ class TestLogzioHandler(TestCase):
                 'message': 'this is a test: moo.',
                 'path_name': 'handler_test.py',
                 'type': 'python'
-             }
+            }
         )
 
     def test_extra_formatting(self):
@@ -105,7 +106,7 @@ class TestLogzioHandler(TestCase):
                 'path_name': 'handler_test.py',
                 'type': 'python',
                 'extra_key': 'extra_value'
-             }
+            }
         )
 
     def test_format_string_message(self):
@@ -133,7 +134,7 @@ class TestLogzioHandler(TestCase):
                 'message': 'this is a test: moo.',
                 'path_name': 'handler_test.py',
                 'type': 'python'
-             }
+            }
         )
 
     def test_exc(self):
@@ -147,7 +148,7 @@ class TestLogzioHandler(TestCase):
             level=0,
             pathname='handler_test.py',
             lineno=10,
-            msg="this is a test: moo.",
+            msg='exception test:',
             args=[],
             exc_info=exc_info,
             func='test_json'
@@ -158,6 +159,7 @@ class TestLogzioHandler(TestCase):
 
         formatted_message["exception"] = formatted_message["exception"].replace(os.path.abspath(__file__), "")
         formatted_message["exception"] = re.sub(r", line \d+", "", formatted_message["exception"])
+        formatted_message["message"] = formatted_message["message"].replace(os.path.abspath(__file__), "")
 
         self.assertDictEqual(
             {
@@ -165,11 +167,12 @@ class TestLogzioHandler(TestCase):
                 'line_number': 10,
                 'log_level': 'NOTSET',
                 'logger': 'my-logger',
-                'message': 'this is a test: moo.',
-                'exception': 'Traceback (most recent call last):\n\n  File "", in test_exc\n    raise ValueError("oops.")\n\nValueError: oops.\n',
+                'message': f'exception test:\nTraceback (most recent call last):\n  File "", line 142, in test_exc\n    raise '
+                           'ValueError("oops.")\nValueError: oops.',
+                'exception': 'Traceback (most recent call last):\n\n  File "", in test_exc\n    raise ValueError('
+                             '"oops.")\n\nValueError: oops.\n',
                 'path_name': 'handler_test.py',
                 'type': 'python'
             },
             formatted_message
         )
-

tests/test_logzioSender.py

@@ -24,6 +24,8 @@ class TestLogzioSender(TestCase):
         self.logzio_listener.clear_logs_buffer()
         self.logzio_listener.clear_server_error()
         self.logs_drain_timeout = 1
+        self.retries_no = 4
+        self.retry_timeout = 2
 
         logging_configuration = {
             "version": 1,
@@ -42,7 +44,9 @@ class TestLogzioSender(TestCase):
                     'logzio_type': "type",
                     'logs_drain_timeout': self.logs_drain_timeout,
                     'url': "http://" + self.logzio_listener.get_host() + ":" + str(self.logzio_listener.get_port()),
-                    'debug': True
+                    'debug': True,
+                    'retries_no': self.retries_no,
+                    'retry_timeout': self.retry_timeout
                 }
             },
             "loggers": {
@@ -88,7 +92,7 @@ class TestLogzioSender(TestCase):
 
         self.logzio_listener.clear_server_error()
 
-        time.sleep(self.logs_drain_timeout * 2 * 4)  # Longer, because of the retry
+        time.sleep(self.logs_drain_timeout * self.retry_timeout * self.retries_no)  # Longer, because of the retry
 
         self.assertTrue(self.logzio_listener.find_log(log_message))
 
@@ -100,7 +104,7 @@ class TestLogzioSender(TestCase):
         # Make sure no file is present
         self.assertEqual(len(_find("logzio-failures-*.txt", ".")), 0)
 
-        time.sleep(2 * 2 * 2 * 2 * 2)  # All of the retries
+        time.sleep(self.retries_no*self.retry_timeout*2*2)  # All of the retries
 
         failure_files = _find("logzio-failures-*.txt", ".")
         self.assertEqual(len(failure_files), 1)
@@ -118,7 +122,7 @@ class TestLogzioSender(TestCase):
         # Make sure no file is present
         self.assertEqual(len(_find("logzio-failures-*.txt", ".")), 0)
 
-        time.sleep(2 * 2 * 2 * 2 * 2)  # All of the retries
+        time.sleep(self.retries_no*self.retry_timeout)  # All of the retries
 
         # Make sure no file was created
         self.assertEqual(len(_find("logzio-failures-*.txt", ".")), 0)