diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..54251d1 Binary files /dev/null and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index a5d7f8c..2fd3190 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__/ # Distribution / packaging .Python +/venv/ /env/ /bin/ /build/ diff --git a/README.md b/README.md index a9b71fd..c08c995 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ In case the logs failed to be sent to Logz.io after a couple of tries, they will pip install logzio-python-handler ``` +If you'd like to use [Trace context](https://github.com/logzio/logzio-python-handler#trace-context) then you need to install the OpenTelemetry logging instrumentation dependecy by running the following command: + +```bash +pip install logzio-python-handler[opentelemetry-logging] +``` ## Tested Python Versions Travis CI will build this handler and test against: - "3.5" @@ -33,6 +38,8 @@ Travis CI will build this handler and test against: - "3.7" - "3.8" - "3.9" + - "3.10" + - "3.11" We can't ensure compatibility to any other version, as we can't test it automatically. @@ -47,7 +54,7 @@ $ tox ## Python configuration #### Config File -``` +```python [handlers] keys=LogzioHandler @@ -84,7 +91,7 @@ format={"additional_field": "value"} i.e. you cannot set Debug to true, without configuring all of the previous parameters as well. #### Dict Config -``` +```python LOGGING = { 'version': 1, 'disable_existing_loggers': False, @@ -154,13 +161,37 @@ Please note, that you cannot override default fields by the python logger (i.e. For example: -``` +```python logger.info('Warning', extra={'extra_key':'extra_value'}) ``` +#### Dynamic Extra Fields +If you prefer, you can add extra fields to your logs dynamically, and not pre-defining them in the configuration. +This way, you can allow different logs to have different extra fields. +See the following code example: + +```python +from logzio.handler import ExtraFieldsLogFilter + +def main(): + + logger.info("Test log") # Outputs: {"message":"Test log"} + + extra_fields = {"foo":"bar","counter":1} + logger.addFilter(ExtraFieldsLogFilter(extra_fields)) + logger.warning("Warning test log") # Outputs: {"message":"Warning test log","foo":"bar","counter":1} + + error_fields = {"err_msg":"Failed to run due to exception.","status_code":500} + logger.addFilter(ExtraFieldsLogFilter(error_fields)) + logger.error("Error test log") # Outputs: {"message":"Error test log","foo":"bar","counter":1,"err_msg":"Failed to run due to exception.","status_code":500} + # If you'd like to remove filters from future logs using the logger.removeFilter option: + logger.removeFilter(ExtraFieldsLogFilter(error_fields)) + logger.debug("Debug test log") # Outputs: {"message":"Debug test log","foo":"bar","counter":1} -## Django configuration ``` + +## Django configuration +```python LOGGING = { 'version': 1, 'disable_existing_loggers': False, @@ -216,6 +247,12 @@ LOGGING = { If you're sending traces with OpenTelemetry instrumentation (auto or manual), you can correlate your logs with the trace context. That way, your logs will have traces data in it, such as service name, span id and trace id. + +Make sure to install the OpenTelemetry logging instrumentation dependecy by running the following command: + +```bash +pip install logzio-python-handler[opentelemetry-logging] +``` To enable this feature, set the `add_context` param in your handler configuration to `True`, like in this example: ```python @@ -255,7 +292,12 @@ LOGGING = { Please note that if you are using `python 3.8`, it is preferred to use the `logging.config.dictConfig` method, as mentioned in [python's documentation](https://docs.python.org/3/library/logging.config.html#configuration-file-format). ## Release Notes - +- 4.1.0 + - Add ability to dynamically attach extra fields to the logs. + - Import opentelemetry logging dependency only if trace context is enabled and dependency is installed manually. + - Updated `opentelemetry-instrumentation-logging==0.39b0` + - Updated `setuptools>=68.0.0` + - Added tests for Python versions: 3.9, 3.10, 3.11 - 4.0.2 - Fix bug for logging exceptions ([#76](https://github.com/logzio/logzio-python-handler/pull/76)) - 4.0.1 diff --git a/logzio/handler.py b/logzio/handler.py index b81dde9..bf17d27 100644 --- a/logzio/handler.py +++ b/logzio/handler.py @@ -8,7 +8,16 @@ import logging.handlers from .sender import LogzioSender from .exceptions import LogzioException -from opentelemetry.instrumentation.logging import LoggingInstrumentor + +class ExtraFieldsLogFilter(logging.Filter): + + def __init__(self, extra: dict, *args, **kwargs): + super().__init__(*args, **kwargs) + self.extra = extra + + def filter(self, record): + record.__dict__.update(self.extra) + return True class LogzioHandler(logging.Handler): @@ -31,8 +40,14 @@ class LogzioHandler(logging.Handler): self.logzio_type = logzio_type if add_context: - LoggingInstrumentor().instrument(set_logging_format=True) - + try: + from opentelemetry.instrumentation.logging import LoggingInstrumentor + LoggingInstrumentor().instrument(set_logging_format=True) + except ImportError: + print("""Can't add trace context. +OpenTelemetry logging optional package isn't installed. +Please install the following package: +pip install 'logzio-python-handler[opentelemetry-logging]'""") self.logzio_sender = LogzioSender( token=token, url=url, @@ -91,7 +106,7 @@ class LogzioHandler(logging.Handler): def format_message(self, message): now = datetime.datetime.utcnow() timestamp = now.strftime('%Y-%m-%dT%H:%M:%S') + \ - '.%03d' % (now.microsecond / 1000) + 'Z' + '.%03d' % (now.microsecond / 1000) + 'Z' return_json = { 'logger': message.name, diff --git a/logzio/sender.py b/logzio/sender.py index 5a7d16c..999b729 100644 --- a/logzio/sender.py +++ b/logzio/sender.py @@ -9,7 +9,6 @@ from threading import Thread, enumerate import requests -from .logger import get_logger from .logger import get_stdout_logger if sys.version[0] == '2': diff --git a/requirements.txt b/requirements.txt index 3169b13..9b4e9a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ requests>=2.27.0 protobuf>=3.20.2 -opentelemetry-instrumentation-logging==0.32b0 -setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability \ No newline at end of file +setuptools>=68.0.0 # not directly required, pinned to avoid a vulnerability \ No newline at end of file diff --git a/setup.py b/setup.py index 00c8b24..5157705 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages setup( name="logzio-python-handler", - version='4.0.2', + version='4.1.0', description="Logging handler to send logs to your Logz.io account with bulk SSL", keywords="logging handler logz.io bulk https", author="roiravhon", @@ -14,9 +14,11 @@ setup( packages=find_packages(), install_requires=[ "requests>=2.27.0", - "protobuf>=3.20.2", - "opentelemetry-instrumentation-logging==0.32b0" + "protobuf>=3.20.2" ], + extras_require={ + "opentelemetry-logging": ["opentelemetry-instrumentation-logging==0.39b0"] + }, test_requires=[ "future" ], diff --git a/tests/test_add_context.py b/tests/test_add_context.py index d77fbf3..d176857 100644 --- a/tests/test_add_context.py +++ b/tests/test_add_context.py @@ -27,8 +27,8 @@ class TestAddContext(TestCase): self.logs_drain_timeout = 1 self.retries_no = 4 self.retry_timeout = 2 - - logging_configuration = { + self.add_context = True + self.logging_configuration = { "version": 1, "formatters": { "logzio": { @@ -48,7 +48,7 @@ class TestAddContext(TestCase): 'debug': True, 'retries_no': self.retries_no, 'retry_timeout': self.retry_timeout, - 'add_context': True + 'add_context': self.add_context } }, "loggers": { @@ -59,13 +59,14 @@ class TestAddContext(TestCase): } } - logging.config.dictConfig(logging_configuration) + logging.config.dictConfig(self.logging_configuration) self.logger = logging.getLogger('test') for curr_file in _find("logzio-failures-*.txt", "."): os.remove(curr_file) def test_add_context(self): + # Logging configuration of add_context default to True log_message = "this log should have a trace context" self.logger.info(log_message) time.sleep(self.logs_drain_timeout * 2) @@ -73,8 +74,28 @@ class TestAddContext(TestCase): for current_log in logs_list: if log_message in current_log: log_dict = json.loads(current_log) - self.assertTrue('otelSpanID' in log_dict) - self.assertTrue('otelTraceID' in log_dict) - self.assertTrue('otelServiceName' in log_dict) - + try: + self.assertTrue('otelSpanID' in log_dict) + self.assertTrue('otelTraceID' in log_dict) + self.assertTrue('otelServiceName' in log_dict) + except AssertionError as err: + print(err) + def test_ignore_context(self): + # Set add_context to False and reconfigure the logger as it defaults to True + self.logging_configuration["handlers"]["LogzioHandler"]["add_context"] = False + logging.config.dictConfig(self.logging_configuration) + self.logger = logging.getLogger('test') + log_message = "this log should not have a trace context" + self.logger.info(log_message) + time.sleep(self.logs_drain_timeout * 2) + logs_list = self.logzio_listener.logs_list + for current_log in logs_list: + if log_message in current_log: + log_dict = json.loads(current_log) + try: + self.assertFalse('otelSpanID' in log_dict) + self.assertFalse('otelTraceID' in log_dict) + self.assertFalse('otelServiceName' in log_dict) + except AssertionError as err: + print(err) diff --git a/tests/test_extra_fields.py b/tests/test_extra_fields.py new file mode 100644 index 0000000..33e9c5b --- /dev/null +++ b/tests/test_extra_fields.py @@ -0,0 +1,134 @@ +import fnmatch +import logging.config +import os +import time +import json +from unittest import TestCase +from logzio.handler import ExtraFieldsLogFilter +from .mockLogzioListener import listener + + +def _find(pattern, path): + result = [] + for root, dirs, files in os.walk(path): + for name in files: + if fnmatch.fnmatch(name, pattern): + result.append(os.path.join(root, name)) + + break # Not descending recursively + return result + + +class TestExtraFieldsFilter(TestCase): + + def setUp(self): + self.logzio_listener = listener.MockLogzioListener() + self.logzio_listener.clear_logs_buffer() + self.logzio_listener.clear_server_error() + self.logs_drain_timeout = 1 + self.retries_no = 4 + self.retry_timeout = 2 + self.add_context = True + logging_configuration = { + "version": 1, + "formatters": { + "logzio": { + "format": '{"key": "value"}', + "validate": False + } + }, + "handlers": { + "LogzioHandler": { + "class": "logzio.handler.LogzioHandler", + "formatter": "logzio", + "level": "DEBUG", + "token": "token", + 'logzio_type': "type", + 'logs_drain_timeout': self.logs_drain_timeout, + 'url': "http://" + self.logzio_listener.get_host() + ":" + str(self.logzio_listener.get_port()), + 'debug': True, + 'retries_no': self.retries_no, + 'retry_timeout': self.retry_timeout, + 'add_context': self.add_context + } + }, + "loggers": { + "test": { + "handlers": ["LogzioHandler"], + "level": "DEBUG" + } + } + } + + logging.config.dictConfig(logging_configuration) + self.logger = logging.getLogger('test') + + for curr_file in _find("logzio-failures-*.txt", "."): + os.remove(curr_file) + + def test_add_extra_fields(self): + extra_fields = {"foo": "bar"} + self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields)) + log_message = "this log should have a additional fields" + self.logger.info(log_message) + time.sleep(self.logs_drain_timeout * 2) + logs_list = self.logzio_listener.logs_list + for current_log in logs_list: + if log_message in current_log: + log_dict = json.loads(current_log) + try: + self.assertEqual(extra_fields, {**extra_fields, **log_dict}) + except AssertionError as err: + print(err) + + def test_remove_extra_fields(self): + extra_fields = {"foo": "bar"} + + self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields)) + log_message = "this log should have a additional fields" + self.logger.info(log_message) + + self.logger.removeFilter(ExtraFieldsLogFilter(extra=extra_fields)) + unfiltered_log_message = "this log shouldn't have a additional fields" + self.logger.info(unfiltered_log_message) + + time.sleep(self.logs_drain_timeout * 2) + logs_list = self.logzio_listener.logs_list + for current_log in logs_list: + if unfiltered_log_message in current_log: + log_dict = json.loads(current_log) + try: + self.assertNotEqual(extra_fields, {**extra_fields, **log_dict}) + except AssertionError as err: + print(err) + + def test_add_multiple_extra_fields(self): + extra_fields = {"foo": "bar"} + self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields)) + log_message = "this log should have additional fields" + self.logger.info(log_message) + + extra_fields = {"counter":1} + self.logger.addFilter(ExtraFieldsLogFilter(extra=extra_fields)) + filtered_log_message = "this log should have multiple additional fields" + self.logger.info(filtered_log_message) + + time.sleep(self.logs_drain_timeout * 2) + logs_list = self.logzio_listener.logs_list + for current_log in logs_list: + if log_message in current_log: + log_dict = json.loads(current_log) + try: + self.assertEqual(extra_fields, {**extra_fields, **log_dict}) + except AssertionError as err: + print(err) + elif filtered_log_message in current_log: + log_dict = json.loads(current_log) + try: + self.assertEqual(extra_fields, {**extra_fields, **log_dict}) + except AssertionError as err: + print(err) + + +if __name__ == '__main__': + unittest.main() diff --git a/tox.ini b/tox.ini index 7335ca1..bf349a1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.7.2 -envlist = flake8, py3flake8, python3.5, python3.6, python3.7, python3.8, python3.8, pypy, pypy3 +envlist = flake8, py3flake8, python3.5, python3.6, python3.7, python3.8, python3.9, python3.10, python3.11, pypy, pypy3 skip_missing_interpreters = true [testenv]