Delete .DS_Store

Change the readme structure

.gitignore

@@ -7,6 +7,7 @@ __pycache__/
 
 # Distribution / packaging
 .Python
+/venv/
 /env/
 /bin/
 /build/

README.md

@@ -26,6 +26,11 @@ In case the logs failed to be sent to Logz.io after a couple of tries, they will
 pip install logzio-python-handler
 ```
 
+If you'd like to use [Trace context](#trace-context), you need to install the OpenTelemetry logging instrumentation dependency by running the following command:
+
+```bash
+pip install logzio-python-handler[opentelemetry-logging]
+```
 ## Tested Python Versions
 Travis CI will build this handler and test against:
   - "3.5"
@@ -33,6 +38,8 @@ Travis CI will build this handler and test against:
   - "3.7"
   - "3.8"
   - "3.9"
+  - "3.10"
+  - "3.11"
 
 We can't ensure compatibility to any other version, as we can't test it automatically.
 
@@ -47,14 +54,16 @@ $ tox
 
 ## Python configuration
 #### Config File
-```
+```python
 [handlers]
 keys=LogzioHandler
 
 [handler_LogzioHandler]
 class=logzio.handler.LogzioHandler
 formatter=logzioFormat
-args=('token', 'my_type')
+
+# Parameters must be set in order. Replace these parameters with your configuration.
+args=('<<LOG-SHIPPING-TOKEN>>', '<<LOG-TYPE>>', <<TIMEOUT>>, 'https://<<LISTENER-HOST>>:8071', <<DEBUG-FLAG>>,<<NETWORKING-TIMEOUT>>,<<RETRY-LIMIT>>,<<RETRY-TIMEOUT>>)
 
 [formatters]
 keys=logzioFormat
@@ -84,7 +93,7 @@ format={"additional_field": "value"}
  i.e. you cannot set Debug to true, without configuring all of the previous parameters as well.
 
 #### Dict Config
-```
+```python
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
@@ -124,14 +133,23 @@ Replace:
 If you're using a serverless function, you'll need to import and add the LogzioFlusher annotation before your sender function. To do this, in the code sample below, uncomment the `import` statement and the `@LogzioFlusher(logger)` annotation line.  
 **Note:** For the LogzioFlusher to work properly, you'll need to make sure that the Logz.io. handler is added to the root logger. See the configuration above for an example.
 
+#### Dynamic Extra Fields
+If you prefer, you can add extra fields to your logs dynamically, and not pre-defining them in the configuration.
+This way, you can allow different logs to have different extra fields.
+Example in the code below. 
+
 #### Code Example
+
 ```python
 import logging
 import logging.config
 # If you're using a serverless function, uncomment.
 # from logzio.flusher import LogzioFlusher
 
-# Say I have saved my dictionary configuration in a variable named 'LOGGING' - see 'Dict Config' sample section
+# If you'd like to leverage the dynamic extra fields feature, uncomment.
+# from logzio.handler import ExtraFieldsLogFilter
+
+# Say I have saved my configuration as a dictionary in a variable named 'LOGGING' - see 'Dict Config' sample section
 logging.config.dictConfig(LOGGING)
 logger = logging.getLogger('superAwesomeLogzioLogger')
 
@@ -145,22 +163,83 @@ def my_func():
         1/0
     except:
         logger.exception("Supporting exceptions too!")
+
+# Example additional code that demonstrates how to dynamically add/remove fields within the code, make sure class is imported.
+
+    logger.info("Test log")  # Outputs: {"message":"Test log"}
+    
+    extra_fields = {"foo":"bar","counter":1}
+    logger.addFilter(ExtraFieldsLogFilter(extra_fields))
+    logger.warning("Warning test log")  # Outputs: {"message":"Warning test log","foo":"bar","counter":1}
+    
+    error_fields = {"err_msg":"Failed to run due to exception.","status_code":500}
+    logger.addFilter(ExtraFieldsLogFilter(error_fields))
+    logger.error("Error test log")  # Outputs: {"message":"Error test log","foo":"bar","counter":1,"err_msg":"Failed to run due to exception.","status_code":500}
+    
+    # If you'd like to remove filters from future logs using the logger.removeFilter option:
+    logger.removeFilter(ExtraFieldsLogFilter(error_fields))
+    logger.debug("Debug test log") # Outputs: {"message":"Debug test log","foo":"bar","counter":1}
+
 ```
 
 #### Extra Fields
-In case you need to dynamic metadata to your logger, other then the constant metadata from the formatter, you can use the "extra" parameter.
+In case you need to dynamic metadata to a speific log and not [dynamically to the logger](#dynamic-extra-fields), other than the constant metadata from the formatter, you can use the "extra" parameter.
 All key values in the dictionary passed in "extra" will be presented in Logz.io as new fields in the log you are sending.
 Please note, that you cannot override default fields by the python logger (i.e. lineno, thread, etc..)
 For example:
 
-
-```
+```python
 logger.info('Warning', extra={'extra_key':'extra_value'})
 ```
 
+#### Trace context
 
-## Django configuration
+If you're sending traces with OpenTelemetry instrumentation (auto or manual), you can correlate your logs with the trace context.
+That way, your logs will have traces data in it, such as service name, span id and trace id.
+
+Make sure to install the OpenTelemetry logging instrumentation dependecy by running the following command:
+
+```shell
+pip install logzio-python-handler[opentelemetry-logging]
 ```
+To enable this feature, set the `add_context` param in your handler configuration to `True`, like in this example:
+
+```python
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'logzioFormat': {
+            'format': '{"additional_field": "value"}',
+            'validate': False
+        }
+    },
+    'handlers': {
+        'logzio': {
+            'class': 'logzio.handler.LogzioHandler',
+            'level': 'INFO',
+            'formatter': 'logzioFormat',
+            'token': '<<LOGZIO-TOKEN>>',
+            'logzio_type': 'python-handler',
+            'logs_drain_timeout': 5,
+            'url': 'https://<<LOGZIO-URL>>:8071',
+            'retries_no': 4,
+            'retry_timeout': 2,
+            'add_context': True
+        }
+    },
+    'loggers': {
+        '': {
+            'level': 'DEBUG',
+            'handlers': ['logzio'],
+            'propagate': True
+        }
+    }
+}
+```
+
+#### Django configuration
+```python
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
@@ -204,57 +283,16 @@ LOGGING = {
 
 ```
 
-*Change*
-- token - Your logzio token
-- url - Logz.io Listener address
-- logs_drain_count - Number of logs to keep in buffer before draining
-- logs_drain_timeout - Time to wait before draining, regardless of the previouse setting
-- logzio_type - Log type, for searching in logz.io (defaults to "python"), it cannot contain a space.
-- appname - Your django app
-
-## Trace context
-
-If you're sending traces with OpenTelemetry instrumentation (auto or manual), you can correlate your logs with the trace context.
-That way, your logs will have traces data in it, such as service name, span id and trace id.
-To enable this feature, set the `add_context` param in your handler configuration to `True`, like in this example:
-
-```python
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'formatters': {
-        'logzioFormat': {
-            'format': '{"additional_field": "value"}',
-            'validate': False
-        }
-    },
-    'handlers': {
-        'logzio': {
-            'class': 'logzio.handler.LogzioHandler',
-            'level': 'INFO',
-            'formatter': 'logzioFormat',
-            'token': '<<LOGZIO-TOKEN>>',
-            'logzio_type': 'python-handler',
-            'logs_drain_timeout': 5,
-            'url': 'https://<<LOGZIO-URL>>:8071',
-            'retries_no': 4,
-            'retry_timeout': 2,
-            'add_context': True
-        }
-    },
-    'loggers': {
-        '': {
-            'level': 'DEBUG',
-            'handlers': ['logzio'],
-            'propagate': True
-        }
-    }
-}
-```
-
-Please note that if you are using `python 3.8`, it is preferred to use the `logging.config.dictConfig` method, as mentioned in [python's documentation](https://docs.python.org/3/library/logging.config.html#configuration-file-format).
 
 ## Release Notes
+- 4.1.0
+  - Add ability to dynamically attach extra fields to the logs.
+  - Import opentelemetry logging dependency only if trace context is enabled and dependency is installed manually.
+    - Updated `opentelemetry-instrumentation-logging==0.39b0`
+  - Updated `setuptools>=68.0.0`
+  - Added tests for Python versions: 3.9, 3.10, 3.11
+- 4.0.2
+  - Fix bug for logging exceptions ([#76](https://github.com/logzio/logzio-python-handler/pull/76))
 - 4.0.1
   - Updated `protobuf>=3.20.2`.
   - Added dependency `setuptools>=65.5.1`
@@ -262,13 +300,15 @@ Please note that if you are using `python 3.8`, it is preferred to use the `logg
 - 4.0.0
   - Add ability to automatically attach trace context to the logs.
 
-- 3.1.1
-  - Bug fixes (issue #68, exception message formatting)
-  - Added CI: Tests and Auto release 
 
 <details>
   <summary markdown="span"> Expand to check old versions </summary>
 
+
+- 3.1.1
+  - Bug fixes (issue #68, exception message formatting)
+  - Added CI: Tests and Auto release
+  
 - 3.1.0
     - Bug fixes
     - Retry number and timeout is now configurable

logzio/handler.py

@@ -8,7 +8,16 @@ import logging.handlers
 from .sender import LogzioSender
 from .exceptions import LogzioException
 
-from opentelemetry.instrumentation.logging import LoggingInstrumentor
+
+class ExtraFieldsLogFilter(logging.Filter):
+
+    def __init__(self, extra: dict, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.extra = extra
+
+    def filter(self, record):
+        record.__dict__.update(self.extra)
+        return True
 
 
 class LogzioHandler(logging.Handler):
@@ -31,8 +40,14 @@ class LogzioHandler(logging.Handler):
         self.logzio_type = logzio_type
 
         if add_context:
-            LoggingInstrumentor().instrument(set_logging_format=True)
-
+            try:
+                from opentelemetry.instrumentation.logging import LoggingInstrumentor
+                LoggingInstrumentor().instrument(set_logging_format=True)
+            except ImportError:
+                print("""Can't add trace context.
+OpenTelemetry logging optional package isn't installed.
+Please install the following package:
+pip install 'logzio-python-handler[opentelemetry-logging]'""")
         self.logzio_sender = LogzioSender(
             token=token,
             url=url,
@@ -79,6 +94,8 @@ class LogzioHandler(logging.Handler):
     def format(self, record):
         message = super(LogzioHandler, self).format(record)
         try:
+            if record.exc_info:
+                message = message.split("\n")[0]  # only keep the original formatted message part
             return json.loads(message)
         except (TypeError, ValueError):
             return message
@@ -89,7 +106,7 @@ class LogzioHandler(logging.Handler):
     def format_message(self, message):
         now = datetime.datetime.utcnow()
         timestamp = now.strftime('%Y-%m-%dT%H:%M:%S') + \
-            '.%03d' % (now.microsecond / 1000) + 'Z'
+                    '.%03d' % (now.microsecond / 1000) + 'Z'
 
         return_json = {
             'logger': message.name,

logzio/sender.py

@@ -9,7 +9,6 @@ from threading import Thread, enumerate
 
 import requests
 
-from .logger import get_logger
 from .logger import get_stdout_logger
 
 if sys.version[0] == '2':

requirements.txt

@@ -1,4 +1,3 @@
 requests>=2.27.0
 protobuf>=3.20.2
-opentelemetry-instrumentation-logging==0.32b0
-setuptools>=65.5.1  # not directly required, pinned by Snyk to avoid a vulnerability
+setuptools>=68.0.0  # not directly required, pinned to avoid a vulnerability

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 setup(
     name="logzio-python-handler",
-    version='4.0.1',
+    version='4.1.0',
     description="Logging handler to send logs to your Logz.io account with bulk SSL",
     keywords="logging handler logz.io bulk https",
     author="roiravhon",
@@ -14,9 +14,11 @@ setup(
     packages=find_packages(),
     install_requires=[
         "requests>=2.27.0",
-        "protobuf>=3.20.2",
-        "opentelemetry-instrumentation-logging==0.32b0"
+        "protobuf>=3.20.2"
     ],
+    extras_require={
+        "opentelemetry-logging": ["opentelemetry-instrumentation-logging==0.39b0"]
+    },
     test_requires=[
         "future"
     ],

tests/test_add_context.py

@@ -27,8 +27,8 @@ class TestAddContext(TestCase):
         self.logs_drain_timeout = 1
         self.retries_no = 4
         self.retry_timeout = 2
-
-        logging_configuration = {
+        self.add_context = True
+        self.logging_configuration = {
             "version": 1,
             "formatters": {
                 "logzio": {
@@ -48,7 +48,7 @@ class TestAddContext(TestCase):
                     'debug': True,
                     'retries_no': self.retries_no,
                     'retry_timeout': self.retry_timeout,
-                    'add_context': True
+                    'add_context': self.add_context
                 }
             },
             "loggers": {
@@ -59,13 +59,14 @@ class TestAddContext(TestCase):
             }
         }
 
-        logging.config.dictConfig(logging_configuration)
+        logging.config.dictConfig(self.logging_configuration)
         self.logger = logging.getLogger('test')
 
         for curr_file in _find("logzio-failures-*.txt", "."):
             os.remove(curr_file)
 
     def test_add_context(self):
+        # Logging configuration of add_context default to True
         log_message = "this log should have a trace context"
         self.logger.info(log_message)
         time.sleep(self.logs_drain_timeout * 2)
@@ -73,8 +74,28 @@ class TestAddContext(TestCase):
         for current_log in logs_list:
             if log_message in current_log:
                 log_dict = json.loads(current_log)
-                self.assertTrue('otelSpanID' in log_dict)
-                self.assertTrue('otelTraceID' in log_dict)
-                self.assertTrue('otelServiceName' in log_dict)
-
+                try:
+                    self.assertTrue('otelSpanID' in log_dict)
+                    self.assertTrue('otelTraceID' in log_dict)
+                    self.assertTrue('otelServiceName' in log_dict)
+                except AssertionError as err:
+                    print(err)
 
+    def test_ignore_context(self):
+        # Set add_context to False and reconfigure the logger as it defaults to True
+        self.logging_configuration["handlers"]["LogzioHandler"]["add_context"] = False
+        logging.config.dictConfig(self.logging_configuration)
+        self.logger = logging.getLogger('test')
+        log_message = "this log should not have a trace context"
+        self.logger.info(log_message)
+        time.sleep(self.logs_drain_timeout * 2)
+        logs_list = self.logzio_listener.logs_list
+        for current_log in logs_list:
+            if log_message in current_log:
+                log_dict = json.loads(current_log)
+                try:
+                    self.assertFalse('otelSpanID' in log_dict)
+                    self.assertFalse('otelTraceID' in log_dict)
+                    self.assertFalse('otelServiceName' in log_dict)
+                except AssertionError as err:
+                    print(err)

tests/test_extra_fields.py

@@ -0,0 +1,134 @@
+import fnmatch
+import logging.config
+import os
+import time
+import json
+from unittest import TestCase
+from logzio.handler import ExtraFieldsLogFilter
+from .mockLogzioListener import listener
+
+
+def _find(pattern, path):
+    result = []
+    for root, dirs, files in os.walk(path):
+        for name in files:
+            if fnmatch.fnmatch(name, pattern):
+                result.append(os.path.join(root, name))
+
+        break  # Not descending recursively
+    return result
+
+
+class TestExtraFieldsFilter(TestCase):
+
+    def setUp(self):
+        self.logzio_listener = listener.MockLogzioListener()
+        self.logzio_listener.clear_logs_buffer()
+        self.logzio_listener.clear_server_error()
+        self.logs_drain_timeout = 1
+        self.retries_no = 4
+        self.retry_timeout = 2
+        self.add_context = True
+        logging_configuration = {
+            "version": 1,
+            "formatters": {
+                "logzio": {
+                    "format": '{"key": "value"}',
+                    "validate": False
+                }
+            },
+            "handlers": {
+                "LogzioHandler": {
+                    "class": "logzio.handler.LogzioHandler",
+                    "formatter": "logzio",
+                    "level": "DEBUG",
+                    "token": "token",
+                    'logzio_type': "type",
+                    'logs_drain_timeout': self.logs_drain_timeout,
+                    'url': "http://" + self.logzio_listener.get_host() + ":" + str(self.logzio_listener.get_port()),
+                    'debug': True,
+                    'retries_no': self.retries_no,
+                    'retry_timeout': self.retry_timeout,
+                    'add_context': self.add_context
+                }
+            },
+            "loggers": {
+                "test": {
+                    "handlers": ["LogzioHandler"],
+                    "level": "DEBUG"
+                }
+            }
+        }
+
+        logging.config.dictConfig(logging_configuration)
+        self.logger = logging.getLogger('test')
+
+        for curr_file in _find("logzio-failures-*.txt", "."):
+            os.remove(curr_file)
+
+    def test_add_extra_fields(self):
+        extra_fields = {"foo": "bar"}
+        self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields))
+        log_message = "this log should have a additional fields"
+        self.logger.info(log_message)
+        time.sleep(self.logs_drain_timeout * 2)
+        logs_list = self.logzio_listener.logs_list
+        for current_log in logs_list:
+            if log_message in current_log:
+                log_dict = json.loads(current_log)
+                try:
+                    self.assertEqual(extra_fields, {**extra_fields, **log_dict})
+                except AssertionError as err:
+                    print(err)
+
+    def test_remove_extra_fields(self):
+        extra_fields = {"foo": "bar"}
+
+        self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields))
+        log_message = "this log should have a additional fields"
+        self.logger.info(log_message)
+
+        self.logger.removeFilter(ExtraFieldsLogFilter(extra=extra_fields))
+        unfiltered_log_message = "this log shouldn't have a additional fields"
+        self.logger.info(unfiltered_log_message)
+
+        time.sleep(self.logs_drain_timeout * 2)
+        logs_list = self.logzio_listener.logs_list
+        for current_log in logs_list:
+            if unfiltered_log_message in current_log:
+                log_dict = json.loads(current_log)
+                try:
+                    self.assertNotEqual(extra_fields, {**extra_fields, **log_dict})
+                except AssertionError as err:
+                    print(err)
+
+    def test_add_multiple_extra_fields(self):
+        extra_fields = {"foo": "bar"}
+        self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields))
+        log_message = "this log should have additional fields"
+        self.logger.info(log_message)
+
+        extra_fields = {"counter":1}
+        self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields))
+        filtered_log_message = "this log should have multiple additional fields"
+        self.logger.info(filtered_log_message)
+
+        time.sleep(self.logs_drain_timeout * 2)
+        logs_list = self.logzio_listener.logs_list
+        for current_log in logs_list:
+            if log_message in current_log:
+                log_dict = json.loads(current_log)
+                try:
+                    self.assertEqual(extra_fields, {**extra_fields, **log_dict})
+                except AssertionError as err:
+                    print(err)
+            elif filtered_log_message in current_log:
+                log_dict = json.loads(current_log)
+                try:
+                    self.assertEqual(extra_fields, {**extra_fields, **log_dict})
+                except AssertionError as err:
+                    print(err)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_logzioHandler.py

@@ -137,7 +137,10 @@ class TestLogzioHandler(TestCase):
             }
         )
 
-    def test_exc(self):
+    def test_exception(self):
+        formatter = logging.Formatter('{"tags": ["staging", "experimental"], "appname": "my-service"}', validate=False)
+        self.handler.setFormatter(formatter)
+        
         try:
             raise ValueError("oops.")
         except:
@@ -163,13 +166,15 @@ class TestLogzioHandler(TestCase):
         self.assertDictEqual(
             {
                 '@timestamp': None,
+                'appname': 'my-service',
                 'line_number': 10,
                 'log_level': 'NOTSET',
                 'logger': 'my-logger',
                 'message': 'exception test:',
-                'exception': 'Traceback (most recent call last):\n\n  File "", in test_exc\n    raise ValueError("oops.")\n\nValueError: oops.\n',
+                'exception': 'Traceback (most recent call last):\n\n  File "", in test_exception\n    raise ValueError("oops.")\n\nValueError: oops.\n',
                 'path_name': 'handler_test.py',
-                'type': 'python'
+                'type': 'python',
+                'tags': ['staging', 'experimental']
             },
             formatted_message
         )

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 minversion = 1.7.2
-envlist = flake8, py3flake8, py35, py36, py37, py38, py39, pypy, pypy3
+envlist = flake8, py3flake8, python3.5, python3.6, python3.7, python3.8, python3.9, python3.10, python3.11, pypy, pypy3
 skip_missing_interpreters = true
 
 [testenv]
@@ -9,7 +9,7 @@ deps =
     requests
     pytest
     pytest-cov
-passenv = CI TRAVIS TRAVIS_*
+passenv = CI,TRAVIS,TRAVIS_*
 commands = pytest --cov-report term-missing --cov logzio tests -v
 
 [testenv:flake8]