diff --git a/.gitignore b/.gitignore index 6416bf8..a5d7f8c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,59 @@ -*.pyc +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] -*egg-info/ +# C extensions +*.so + +# Distribution / packaging +.Python +/env/ +/bin/ +/build/ +/build_docs/ +/develop-eggs/ +/dist/ +/eggs/ +/lib/ +/lib64/ +/parts/ +/sdist/ +/var/ +/*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +/htmlcov/ +/.tox/ +.coverage +.cache +nosetests.xml +coverage.xml +.pytest_cache/ + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +# Sphinx documentation +/docs/_build/ + +*.iml + +*COMMIT_MSG diff --git a/.travis.yml b/.travis.yml index 6456a56..ac94a88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,24 @@ +sudo: false language: python -python: - - "2.7" - - "3.3" - - "3.4" - - "3.5" - - "3.6" -install: - - pip install requests future +matrix: + include: + - python: 2.7 + env: TOXENV=flake8 + - python: 3.6 + env: TOXENV=py3flake8 + - python: 2.7 + env: TOXENV=py27 + - python: 3.3 + env: TOXENV=py33 + - python: 3.4 + env: TOXENV=py34 + - python: 3.5 + env: TOXENV=py35 + - python: 3.6 + env: TOXENV=py36 -script: py.test \ No newline at end of file +install: + - pip install tox +script: + - tox diff --git a/README.md b/README.md index 52eac93..b364422 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ [![PyPI version](https://badge.fury.io/py/logzio-python-handler.svg)](https://badge.fury.io/py/logzio-python-handler) [![Build Status](https://travis-ci.org/logzio/logzio-python-handler.svg?branch=master)](https://travis-ci.org/logzio/logzio-python-handler) # The Logz.io Python Handler -This is a Python handler that sends logs in bulk over HTTPS to Logz.io. -The handler uses a subclass named LogzioSender (which can be used without this handler as well, to ship raw data). -The LogzioSender class opens a new Thread, that consumes from the logs queue. Each iteration (its frequency of which can be configured by the logs_drain_timeout parameter), will try to consume the queue in its entirety. -Logs will get divided into separate bulks, based on their size. -LogzioSender will check if the main thread is alive. In case the main thread quits, it will try to consume the queue one last time, and then exit. So your program can hang for a few seconds, until the logs are drained. +This is a Python handler that sends logs in bulk over HTTPS to Logz.io. +The handler uses a subclass named LogzioSender (which can be used without this handler as well, to ship raw data). +The LogzioSender class opens a new Thread, that consumes from the logs queue. Each iteration (its frequency of which can be configured by the logs_drain_timeout parameter), will try to consume the queue in its entirety. +Logs will get divided into separate bulks, based on their size. +LogzioSender will check if the main thread is alive. In case the main thread quits, it will try to consume the queue one last time, and then exit. So your program can hang for a few seconds, until the logs are drained. In case the logs failed to be sent to Logz.io after a couple of tries, they will be written to the local file system. You can later upload them to Logz.io using curl. ## Installation @@ -20,9 +20,18 @@ Travis CI will build this handler and test against: - "3.4" - "3.5" - "3.6" - + We can't ensure compatibility to any other version, as we can't test it automatically. +To run tests: + +```bash +$ pip install tox +$ tox +... + +``` + ## Python configuration #### Config File ``` @@ -53,8 +62,8 @@ format={"additional_field": "value"} - Time to sleep between draining attempts (defaults to "3") - Logz.io Listener address (defaults to "https://listener.logz.io:8071") - Debug flag. Set to True, will print debug messages to stdout. (defaults to "False") - - Please note, that you have to configure those parameters by this exact order. + + Please note, that you have to configure those parameters by this exact order. i.e. you cannot set Debug to true, without configuring all of the previous parameters as well. #### Code Example @@ -76,10 +85,10 @@ except: ``` #### Extra Fields -In case you need to dynamic metadata to your logger, other then the constant metadata from the formatter, you can use the "extra" parameter. -All key values in the dictionary passed in "extra" will be presented in Logz.io as new fields in the log you are sending. -Please note, that you cannot override default fields by the python logger (i.e. lineno, thread, etc..) -For example: +In case you need to dynamic metadata to your logger, other then the constant metadata from the formatter, you can use the "extra" parameter. +All key values in the dictionary passed in "extra" will be presented in Logz.io as new fields in the log you are sending. +Please note, that you cannot override default fields by the python logger (i.e. lineno, thread, etc..) +For example: ``` @@ -145,7 +154,7 @@ LOGGING = { - 2.0.3 - Fix bug that consumed more logs while draining than Logz.io's bulk limit - 2.0.2 - Support for formatted messages (Thanks @johnraz!) - 2.0.1 - Added __all__ to __init__.py, so support * imports -- 2.0.0 - Production, stable release. +- 2.0.0 - Production, stable release. - *BREAKING* - Configuration option logs_drain_count was removed, and the order of the parameters has changed for better simplicity. Please review the parameters section above. - Introducing the LogzioSender class, which is generic and can be used without the handler wrap to ship raw data to Logz.io. Just create a new instance of the class, and use the append() method. - Simplifications and Robustness diff --git a/logzio/exceptions.py b/logzio/exceptions.py new file mode 100644 index 0000000..9453722 --- /dev/null +++ b/logzio/exceptions.py @@ -0,0 +1,2 @@ +class LogzioException(Exception): + pass diff --git a/logzio/handler.py b/logzio/handler.py index 71f55f8..297df5b 100644 --- a/logzio/handler.py +++ b/logzio/handler.py @@ -1,36 +1,47 @@ -import datetime +import sys import json import logging -import logging.handlers +import datetime import traceback -import sys +import logging.handlers from .sender import LogzioSender +from .exceptions import LogzioException class LogzioHandler(logging.Handler): - def __init__(self, token, logzio_type="python", logs_drain_timeout=3, - url="https://listener.logz.io:8071", debug=False): + def __init__(self, + token, + logzio_type="python", + logs_drain_timeout=3, + url="https://listener.logz.io:8071", + debug=False): - if token is "": - raise Exception("Logz.io Token must be provided") + if not token: + raise LogzioException('Logz.io Token must be provided') self.logzio_type = logzio_type - self.logzio_sender = LogzioSender(token=token, url=url, logs_drain_timeout=logs_drain_timeout, debug=debug) + self.logzio_sender = LogzioSender( + token=token, + url=url, + logs_drain_timeout=logs_drain_timeout, + debug=debug) logging.Handler.__init__(self) def extra_fields(self, message): not_allowed_keys = ( - 'args', 'asctime', 'created', 'exc_info', 'stack_info', 'exc_text', 'filename', - 'funcName', 'levelname', 'levelno', 'lineno', 'module', + 'args', 'asctime', 'created', 'exc_info', 'stack_info', 'exc_text', + 'filename', 'funcName', 'levelname', 'levelno', 'lineno', 'module', 'msecs', 'msecs', 'message', 'msg', 'name', 'pathname', 'process', 'processName', 'relativeCreated', 'thread', 'threadName') if sys.version_info < (3, 0): - var_type = (basestring, bool, dict, float, int, long, list, type(None)) + # long and basestring don't exist in py3 so, NOQA + var_type = (basestring, bool, dict, float, # NOQA + int, long, list, type(None)) # NOQA else: var_type = (str, bool, dict, float, int, list, type(None)) @@ -60,29 +71,29 @@ class LogzioHandler(logging.Handler): def format_message(self, message): now = datetime.datetime.utcnow() - timestamp = now.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % (now.microsecond / 1000) + "Z" + timestamp = now.strftime('%Y-%m-%dT%H:%M:%S') + \ + '.%03d' % (now.microsecond / 1000) + 'Z' return_json = { - "logger": message.name, - "line_number": message.lineno, - "path_name": message.pathname, - "log_level": message.levelname, - "type": self.logzio_type, - "message": message.getMessage(), - "@timestamp": timestamp + 'logger': message.name, + 'line_number': message.lineno, + 'path_name': message.pathname, + 'log_level': message.levelname, + 'type': self.logzio_type, + 'message': message.getMessage(), + '@timestamp': timestamp } if message.exc_info: - return_json["exception"] = self.format_exception(message.exc_info) + return_json['exception'] = self.format_exception(message.exc_info) else: formatted_message = self.format(message) - return_json.update(self.extra_fields(message)) if isinstance(formatted_message, dict): return_json.update(formatted_message) else: - return_json["message"] = formatted_message + return_json['message'] = formatted_message return return_json diff --git a/logzio/logger.py b/logzio/logger.py new file mode 100644 index 0000000..0dcacca --- /dev/null +++ b/logzio/logger.py @@ -0,0 +1,9 @@ +import sys +import logging + + +def get_logger(debug): + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG if debug else logging.INFO) + logger.addHandler(logging.StreamHandler(sys.stdout)) + return logger diff --git a/logzio/sender.py b/logzio/sender.py index eb88a5a..7b51d53 100644 --- a/logzio/sender.py +++ b/logzio/sender.py @@ -1,36 +1,46 @@ -# This class is responsible for handling all asynchronous Logz.io's communication +# This class is responsible for handling all asynchronous Logz.io's +# communication import sys -import requests import json -from threading import Thread, enumerate -from datetime import datetime + from time import sleep +from datetime import datetime +from threading import Thread, enumerate + +import requests + +from .logger import get_logger if sys.version[0] == '2': import Queue as queue else: import queue as queue + MAX_BULK_SIZE_IN_BYTES = 1 * 1024 * 1024 # 1 MB -def backup_logs(logs): - timestamp = datetime.now().strftime("%d%m%Y-%H%M%S") - print("Backing up your logs to logzio-failures-{0}.txt".format(timestamp)) - with open("logzio-failures-{0}.txt".format(timestamp), "a") as f: +def backup_logs(logs, logger): + timestamp = datetime.now().strftime('%d%m%Y-%H%M%S') + logger.info( + 'Backing up your logs to logzio-failures-%s.txt', timestamp) + with open('logzio-failures-{}.txt'.format(timestamp), 'a') as f: f.writelines('\n'.join(logs)) class LogzioSender: - - def __init__(self, token, url="https://listener.logz.io:8071", logs_drain_timeout=5, debug=False): + def __init__(self, + token, url='https://listener.logz.io:8071', + logs_drain_timeout=5, + debug=False): self.token = token - self.url = "{0}/?token={1}".format(url, token) + self.url = '{}/?token={}'.format(url, token) self.logs_drain_timeout = logs_drain_timeout - self.debug = debug + self.logger = get_logger(debug) # Function to see if the main thread is alive - self.is_main_thread_active = lambda: any((i.name == "MainThread") and i.is_alive() for i in enumerate()) + self.is_main_thread_active = lambda: any( + (i.name == 'MainThread') and i.is_alive() for i in enumerate()) # Create a queue to hold logs self.queue = queue.Queue() @@ -39,7 +49,7 @@ class LogzioSender: def _initialize_sending_thread(self): self.sending_thread = Thread(target=self._drain_queue) self.sending_thread.daemon = False - self.sending_thread.name = "logzio-sending-thread" + self.sending_thread.name = 'logzio-sending-thread' self.sending_thread.start() def append(self, logs_message): @@ -52,24 +62,24 @@ class LogzioSender: def flush(self): self._flush_queue() - def _debug(self, message): - if self.debug: - print(str(message)) - def _drain_queue(self): last_try = False while not last_try: - # If main is exited, we should run one last time and try to remove all logs + # If main is exited, we should run one last time and try to remove + # all logs if not self.is_main_thread_active(): - self._debug("Identified quit of main thread, sending logs one last time") + self.logger.debug( + 'Identified quit of main thread, sending logs one ' + 'last time') last_try = True try: self._flush_queue() - except Exception as e: - self._debug("Unexpected exception while draining queue to Logz.io, swallowing. Exception: " + str(e)) + self.logger.debug( + 'Unexpected exception while draining queue to Logz.io, ' + 'swallowing. Exception: %s', e) if not last_try: sleep(self.logs_drain_timeout) @@ -78,7 +88,8 @@ class LogzioSender: # Sending logs until queue is empty while not self.queue.empty(): logs_list = self._get_messages_up_to_max_allowed_size() - self._debug("Starting to drain " + str(len(logs_list)) + " logs to Logz.io") + self.logger.debug( + 'Starting to drain %s logs to Logz.io', len(logs_list)) # Not configurable from the outside sleep_between_retries = 2 @@ -90,27 +101,43 @@ class LogzioSender: for current_try in range(number_of_retries): should_retry = False try: - response = requests.post(self.url, headers=headers, data='\n'.join(logs_list)) + response = requests.post( + self.url, headers=headers, data='\n'.join(logs_list)) if response.status_code != 200: if response.status_code == 400: - print("Got 400 code from Logz.io. This means that some of your logs are too big, or badly formatted. response: {0}".format(response.text)) + self.logger.info( + 'Got 400 code from Logz.io. This means that ' + 'some of your logs are too big, or badly ' + 'formatted. response: %s', response.text) should_backup_to_disk = False break if response.status_code == 401: - print("You are not authorized with Logz.io! Token OK? dropping logs...") + self.logger.info( + 'You are not authorized with Logz.io! Token ' + 'OK? dropping logs...') should_backup_to_disk = False break else: - print("Got {} while sending logs to Logz.io, Try ({}/{}). Response: {}".format(response.status_code, current_try + 1, number_of_retries, response.text)) + self.logger.info( + 'Got %s while sending logs to Logz.io, ' + 'Try (%s/%s). Response: %s', + response.status_code, + current_try + 1, + number_of_retries, + response.text) should_retry = True else: - self._debug("Successfully sent bulk of " + str(len(logs_list)) + " logs to Logz.io!") + self.logger.debug( + 'Successfully sent bulk of %s logs to ' + 'Logz.io!', len(logs_list)) should_backup_to_disk = False break - except Exception as e: - print("Got exception while sending logs to Logz.io, Try ({}/{}). Message: {}".format(current_try + 1, number_of_retries, e)) + self.logger.error( + 'Got exception while sending logs to Logz.io, ' + 'Try (%s/%s). Message: %s', + current_try + 1, number_of_retries, e) should_retry = True if should_retry: @@ -119,8 +146,10 @@ class LogzioSender: if should_backup_to_disk: # Write to file - print("Could not send logs to Logz.io after " + str(number_of_retries) + " tries, backing up to local file system.") - backup_logs(logs_list) + self.logger.info( + 'Could not send logs to Logz.io after %s tries, ' + 'backing up to local file system', number_of_retries) + backup_logs(logs_list, self.logger) def _get_messages_up_to_max_allowed_size(self): logs_list = [] diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..e45862c --- /dev/null +++ b/tox.ini @@ -0,0 +1,23 @@ +[tox] +minversion = 1.7.2 +envlist = flake8, py3flake8, py27, py33, py34, py35, py36 +skip_missing_interpreters = true + +[testenv] +deps = + future + requests + pytest + pytest-cov +passenv = CI TRAVIS TRAVIS_* +commands = pytest --cov-report term-missing --cov logzio tests -v + +[testenv:flake8] +basepython = python2.7 +deps = flake8 +commands = flake8 logzio + +[testenv:py3flake8] +basepython = python3.6 +deps = flake8 +commands = flake8 logzio