Merge pull request #82 from logzio/INT-834

Update dependencies, add optional trace context import & enable dynamic extra fields

.gitignore

@@ -7,6 +7,7 @@ __pycache__/
 
 # Distribution / packaging
 .Python
+/venv/
 /env/
 /bin/
 /build/

README.md

@@ -26,6 +26,11 @@ In case the logs failed to be sent to Logz.io after a couple of tries, they will
 pip install logzio-python-handler
 ```
 
+If you'd like to use [Trace context](https://github.com/logzio/logzio-python-handler#trace-context) then you need to install the OpenTelemetry logging instrumentation dependecy by running the following command:
+
+```bash
+pip install logzio-python-handler[opentelemetry-logging]
+```
 ## Tested Python Versions
 Travis CI will build this handler and test against:
   - "3.5"
@@ -33,6 +38,8 @@ Travis CI will build this handler and test against:
   - "3.7"
   - "3.8"
   - "3.9"
+  - "3.10"
+  - "3.11"
 
 We can't ensure compatibility to any other version, as we can't test it automatically.
 
@@ -47,7 +54,7 @@ $ tox
 
 ## Python configuration
 #### Config File
-```
+```python
 [handlers]
 keys=LogzioHandler
 
@@ -84,7 +91,7 @@ format={"additional_field": "value"}
  i.e. you cannot set Debug to true, without configuring all of the previous parameters as well.
 
 #### Dict Config
-```
+```python
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
@@ -154,13 +161,37 @@ Please note, that you cannot override default fields by the python logger (i.e.
 For example:
 
 
-```
+```python
 logger.info('Warning', extra={'extra_key':'extra_value'})
 ```
+#### Dynamic Extra Fields
+If you prefer, you can add extra fields to your logs dynamically, and not pre-defining them in the configuration.
+This way, you can allow different logs to have different extra fields.
 
+See the following code example:
+
+```python
+from logzio.handler import ExtraFieldsLogFilter
+
+def main():
+
+    logger.info("Test log")  # Outputs: {"message":"Test log"}
+    
+    extra_fields = {"foo":"bar","counter":1}
+    logger.addFilter(ExtraFieldsLogFilter(extra_fields))
+    logger.warning("Warning test log")  # Outputs: {"message":"Warning test log","foo":"bar","counter":1}
+    
+    error_fields = {"err_msg":"Failed to run due to exception.","status_code":500}
+    logger.addFilter(ExtraFieldsLogFilter(error_fields))
+    logger.error("Error test log")  # Outputs: {"message":"Error test log","foo":"bar","counter":1,"err_msg":"Failed to run due to exception.","status_code":500}
+    # If you'd like to remove filters from future logs using the logger.removeFilter option:
+    logger.removeFilter(ExtraFieldsLogFilter(error_fields))
+    logger.debug("Debug test log") # Outputs: {"message":"Debug test log","foo":"bar","counter":1}
 
-## Django configuration
 ```
+
+## Django configuration
+```python
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
@@ -216,6 +247,12 @@ LOGGING = {
 
 If you're sending traces with OpenTelemetry instrumentation (auto or manual), you can correlate your logs with the trace context.
 That way, your logs will have traces data in it, such as service name, span id and trace id.
+
+Make sure to install the OpenTelemetry logging instrumentation dependecy by running the following command:
+
+```bash
+pip install logzio-python-handler[opentelemetry-logging]
+```
 To enable this feature, set the `add_context` param in your handler configuration to `True`, like in this example:
 
 ```python
@@ -255,7 +292,12 @@ LOGGING = {
 Please note that if you are using `python 3.8`, it is preferred to use the `logging.config.dictConfig` method, as mentioned in [python's documentation](https://docs.python.org/3/library/logging.config.html#configuration-file-format).
 
 ## Release Notes
-
+- 4.1.0
+  - Add ability to dynamically attach extra fields to the logs.
+  - Import opentelemetry logging dependency only if trace context is enabled and dependency is installed manually.
+    - Updated `opentelemetry-instrumentation-logging==0.39b0`
+  - Updated `setuptools>=68.0.0`
+  - Added tests for Python versions: 3.9, 3.10, 3.11
 - 4.0.2
   - Fix bug for logging exceptions ([#76](https://github.com/logzio/logzio-python-handler/pull/76))
 - 4.0.1

logzio/handler.py

@@ -8,7 +8,16 @@ import logging.handlers
 from .sender import LogzioSender
 from .exceptions import LogzioException
 
-from opentelemetry.instrumentation.logging import LoggingInstrumentor
+
+class ExtraFieldsLogFilter(logging.Filter):
+
+    def __init__(self, extra: dict, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.extra = extra
+
+    def filter(self, record):
+        record.__dict__.update(self.extra)
+        return True
 
 
 class LogzioHandler(logging.Handler):
@@ -31,8 +40,14 @@ class LogzioHandler(logging.Handler):
         self.logzio_type = logzio_type
 
         if add_context:
-            LoggingInstrumentor().instrument(set_logging_format=True)
-
+            try:
+                from opentelemetry.instrumentation.logging import LoggingInstrumentor
+                LoggingInstrumentor().instrument(set_logging_format=True)
+            except ImportError:
+                print("""Can't add trace context.
+OpenTelemetry logging optional package isn't installed.
+Please install the following package:
+pip install 'logzio-python-handler[opentelemetry-logging]'""")
         self.logzio_sender = LogzioSender(
             token=token,
             url=url,
@@ -91,7 +106,7 @@ class LogzioHandler(logging.Handler):
     def format_message(self, message):
         now = datetime.datetime.utcnow()
         timestamp = now.strftime('%Y-%m-%dT%H:%M:%S') + \
-            '.%03d' % (now.microsecond / 1000) + 'Z'
+                    '.%03d' % (now.microsecond / 1000) + 'Z'
 
         return_json = {
             'logger': message.name,

logzio/sender.py

@@ -9,7 +9,6 @@ from threading import Thread, enumerate
 
 import requests
 
-from .logger import get_logger
 from .logger import get_stdout_logger
 
 if sys.version[0] == '2':

requirements.txt

@@ -1,4 +1,3 @@
 requests>=2.27.0
 protobuf>=3.20.2
-opentelemetry-instrumentation-logging==0.32b0
-setuptools>=65.5.1  # not directly required, pinned by Snyk to avoid a vulnerability
+setuptools>=68.0.0  # not directly required, pinned to avoid a vulnerability

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 setup(
     name="logzio-python-handler",
-    version='4.0.2',
+    version='4.1.0',
     description="Logging handler to send logs to your Logz.io account with bulk SSL",
     keywords="logging handler logz.io bulk https",
     author="roiravhon",
@@ -14,9 +14,11 @@ setup(
     packages=find_packages(),
     install_requires=[
         "requests>=2.27.0",
-        "protobuf>=3.20.2",
-        "opentelemetry-instrumentation-logging==0.32b0"
+        "protobuf>=3.20.2"
     ],
+    extras_require={
+        "opentelemetry-logging": ["opentelemetry-instrumentation-logging==0.39b0"]
+    },
     test_requires=[
         "future"
     ],

tests/test_add_context.py

@@ -27,8 +27,8 @@ class TestAddContext(TestCase):
         self.logs_drain_timeout = 1
         self.retries_no = 4
         self.retry_timeout = 2
-
-        logging_configuration = {
+        self.add_context = True
+        self.logging_configuration = {
             "version": 1,
             "formatters": {
                 "logzio": {
@@ -48,7 +48,7 @@ class TestAddContext(TestCase):
                     'debug': True,
                     'retries_no': self.retries_no,
                     'retry_timeout': self.retry_timeout,
-                    'add_context': True
+                    'add_context': self.add_context
                 }
             },
             "loggers": {
@@ -59,13 +59,14 @@ class TestAddContext(TestCase):
             }
         }
 
-        logging.config.dictConfig(logging_configuration)
+        logging.config.dictConfig(self.logging_configuration)
         self.logger = logging.getLogger('test')
 
         for curr_file in _find("logzio-failures-*.txt", "."):
             os.remove(curr_file)
 
     def test_add_context(self):
+        # Logging configuration of add_context default to True
         log_message = "this log should have a trace context"
         self.logger.info(log_message)
         time.sleep(self.logs_drain_timeout * 2)
@@ -73,8 +74,28 @@ class TestAddContext(TestCase):
         for current_log in logs_list:
             if log_message in current_log:
                 log_dict = json.loads(current_log)
-                self.assertTrue('otelSpanID' in log_dict)
-                self.assertTrue('otelTraceID' in log_dict)
-                self.assertTrue('otelServiceName' in log_dict)
-
+                try:
+                    self.assertTrue('otelSpanID' in log_dict)
+                    self.assertTrue('otelTraceID' in log_dict)
+                    self.assertTrue('otelServiceName' in log_dict)
+                except AssertionError as err:
+                    print(err)
 
+    def test_ignore_context(self):
+        # Set add_context to False and reconfigure the logger as it defaults to True
+        self.logging_configuration["handlers"]["LogzioHandler"]["add_context"] = False
+        logging.config.dictConfig(self.logging_configuration)
+        self.logger = logging.getLogger('test')
+        log_message = "this log should not have a trace context"
+        self.logger.info(log_message)
+        time.sleep(self.logs_drain_timeout * 2)
+        logs_list = self.logzio_listener.logs_list
+        for current_log in logs_list:
+            if log_message in current_log:
+                log_dict = json.loads(current_log)
+                try:
+                    self.assertFalse('otelSpanID' in log_dict)
+                    self.assertFalse('otelTraceID' in log_dict)
+                    self.assertFalse('otelServiceName' in log_dict)
+                except AssertionError as err:
+                    print(err)

tests/test_extra_fields.py

@@ -0,0 +1,134 @@
+import fnmatch
+import logging.config
+import os
+import time
+import json
+from unittest import TestCase
+from logzio.handler import ExtraFieldsLogFilter
+from .mockLogzioListener import listener
+
+
+def _find(pattern, path):
+    result = []
+    for root, dirs, files in os.walk(path):
+        for name in files:
+            if fnmatch.fnmatch(name, pattern):
+                result.append(os.path.join(root, name))
+
+        break  # Not descending recursively
+    return result
+
+
+class TestExtraFieldsFilter(TestCase):
+
+    def setUp(self):
+        self.logzio_listener = listener.MockLogzioListener()
+        self.logzio_listener.clear_logs_buffer()
+        self.logzio_listener.clear_server_error()
+        self.logs_drain_timeout = 1
+        self.retries_no = 4
+        self.retry_timeout = 2
+        self.add_context = True
+        logging_configuration = {
+            "version": 1,
+            "formatters": {
+                "logzio": {
+                    "format": '{"key": "value"}',
+                    "validate": False
+                }
+            },
+            "handlers": {
+                "LogzioHandler": {
+                    "class": "logzio.handler.LogzioHandler",
+                    "formatter": "logzio",
+                    "level": "DEBUG",
+                    "token": "token",
+                    'logzio_type': "type",
+                    'logs_drain_timeout': self.logs_drain_timeout,
+                    'url': "http://" + self.logzio_listener.get_host() + ":" + str(self.logzio_listener.get_port()),
+                    'debug': True,
+                    'retries_no': self.retries_no,
+                    'retry_timeout': self.retry_timeout,
+                    'add_context': self.add_context
+                }
+            },
+            "loggers": {
+                "test": {
+                    "handlers": ["LogzioHandler"],
+                    "level": "DEBUG"
+                }
+            }
+        }
+
+        logging.config.dictConfig(logging_configuration)
+        self.logger = logging.getLogger('test')
+
+        for curr_file in _find("logzio-failures-*.txt", "."):
+            os.remove(curr_file)
+
+    def test_add_extra_fields(self):
+        extra_fields = {"foo": "bar"}
+        self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields))
+        log_message = "this log should have a additional fields"
+        self.logger.info(log_message)
+        time.sleep(self.logs_drain_timeout * 2)
+        logs_list = self.logzio_listener.logs_list
+        for current_log in logs_list:
+            if log_message in current_log:
+                log_dict = json.loads(current_log)
+                try:
+                    self.assertEqual(extra_fields, {**extra_fields, **log_dict})
+                except AssertionError as err:
+                    print(err)
+
+    def test_remove_extra_fields(self):
+        extra_fields = {"foo": "bar"}
+
+        self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields))
+        log_message = "this log should have a additional fields"
+        self.logger.info(log_message)
+
+        self.logger.removeFilter(ExtraFieldsLogFilter(extra=extra_fields))
+        unfiltered_log_message = "this log shouldn't have a additional fields"
+        self.logger.info(unfiltered_log_message)
+
+        time.sleep(self.logs_drain_timeout * 2)
+        logs_list = self.logzio_listener.logs_list
+        for current_log in logs_list:
+            if unfiltered_log_message in current_log:
+                log_dict = json.loads(current_log)
+                try:
+                    self.assertNotEqual(extra_fields, {**extra_fields, **log_dict})
+                except AssertionError as err:
+                    print(err)
+
+    def test_add_multiple_extra_fields(self):
+        extra_fields = {"foo": "bar"}
+        self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields))
+        log_message = "this log should have additional fields"
+        self.logger.info(log_message)
+
+        extra_fields = {"counter":1}
+        self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields))
+        filtered_log_message = "this log should have multiple additional fields"
+        self.logger.info(filtered_log_message)
+
+        time.sleep(self.logs_drain_timeout * 2)
+        logs_list = self.logzio_listener.logs_list
+        for current_log in logs_list:
+            if log_message in current_log:
+                log_dict = json.loads(current_log)
+                try:
+                    self.assertEqual(extra_fields, {**extra_fields, **log_dict})
+                except AssertionError as err:
+                    print(err)
+            elif filtered_log_message in current_log:
+                log_dict = json.loads(current_log)
+                try:
+                    self.assertEqual(extra_fields, {**extra_fields, **log_dict})
+                except AssertionError as err:
+                    print(err)
+
+
+if __name__ == '__main__':
+    unittest.main()

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 minversion = 1.7.2
-envlist = flake8, py3flake8, python3.5, python3.6, python3.7, python3.8, python3.8, pypy, pypy3
+envlist = flake8, py3flake8, python3.5, python3.6, python3.7, python3.8, python3.9, python3.10, python3.11, pypy, pypy3
 skip_missing_interpreters = true
 
 [testenv]