zolfa
/
opensearch-python-handler
1
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 0 Wiki Activity
Browse Source

bug fixes

opensearch
tamir-michaeli 3 years ago
parent
9fda10efd3
commit
e7f9585b6c
7 changed files with 65 additions and 41 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +8
    -6
      logzio/handler.py
  2. +11
    -2
      logzio/logger.py
  3. +22
    -17
      logzio/sender.py
  4. +1
    -1
      requirements.txt
  5. +4
    -3
      setup.py
  6. +11
    -8
      tests/test_logzioHandler.py
  7. +8
    -4
      tests/test_logzioSender.py

+ 8
- 6
logzio/handler.py View File

@ -18,7 +18,9 @@ class LogzioHandler(logging.Handler):
url="https://listener.logz.io:8071",
debug=False,
backup_logs=True,
network_timeout=10.0):
network_timeout=10.0,
retries_no=4,
retry_timeout=2):
if not token:
raise LogzioException('Logz.io Token must be provided')
@ -31,7 +33,9 @@ class LogzioHandler(logging.Handler):
logs_drain_timeout=logs_drain_timeout,
debug=debug,
backup_logs=backup_logs,
network_timeout=network_timeout)
network_timeout=network_timeout,
number_of_retries=retries_no,
retry_timeout=retry_timeout)
logging.Handler.__init__(self)
def __del__(self):
@ -94,10 +98,8 @@ class LogzioHandler(logging.Handler):
if message.exc_info:
return_json['exception'] = self.format_exception(message.exc_info)
# We want to ignore default logging formatting on exceptions
# As we handle those differently directly into exception field
message.exc_info = None
message.exc_text = None
# # We want to ignore default logging formatting on exceptions
# # As we handle those differently directly into exception field
formatted_message = self.format(message)
if isinstance(formatted_message, dict):


+ 11
- 2
logzio/logger.py View File

@ -3,7 +3,16 @@ import logging
def get_logger(debug):
logger = logging.getLogger(__name__)
return __get_logger(debug, __name__)
def get_stdout_logger(debug):
return __get_logger(debug, __name__ + '_stdout', logging.StreamHandler(sys.stdout))
def __get_logger(debug, name, handler=None):
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG if debug else logging.INFO)
logger.addHandler(logging.StreamHandler(sys.stdout))
if handler:
logger.addHandler(handler)
return logger

+ 22
- 17
logzio/sender.py View File

@ -10,13 +10,13 @@ from threading import Thread, enumerate
import requests
from .logger import get_logger
from .logger import get_stdout_logger
if sys.version[0] == '2':
import Queue as queue
else:
import queue as queue
MAX_BULK_SIZE_IN_BYTES = 1 * 1024 * 1024 # 1 MB
@ -34,14 +34,19 @@ class LogzioSender:
logs_drain_timeout=5,
debug=False,
backup_logs=True,
network_timeout=10.0):
network_timeout=10.0,
number_of_retries=4,
retry_timeout=2):
self.token = token
self.url = '{}/?token={}'.format(url, token)
self.logs_drain_timeout = logs_drain_timeout
self.logger = get_logger(debug)
self.stdout_logger = get_stdout_logger(debug)
self.backup_logs = backup_logs
self.network_timeout = network_timeout
self.requests_session = requests.Session()
self.number_of_retries = number_of_retries
self.retry_timeout = retry_timeout
# Function to see if the main thread is alive
self.is_main_thread_active = lambda: any(
@ -53,6 +58,7 @@ class LogzioSender:
def __del__(self):
del self.logger
del self.stdout_logger
del self.backup_logs
del self.queue
@ -79,7 +85,7 @@ class LogzioSender:
# If main is exited, we should run one last time and try to remove
# all logs
if not self.is_main_thread_active():
self.logger.debug(
self.stdout_logger.debug(
'Identified quit of main thread, sending logs one '
'last time')
last_try = True
@ -98,17 +104,17 @@ class LogzioSender:
# Sending logs until queue is empty
while not self.queue.empty():
logs_list = self._get_messages_up_to_max_allowed_size()
self.logger.debug(
self.stdout_logger.debug(
'Starting to drain %s logs to Logz.io', len(logs_list))
# Not configurable from the outside
sleep_between_retries = 2
number_of_retries = 4
sleep_between_retries = self.retry_timeout
self.number_of_retries = self.number_of_retries
should_backup_to_disk = True
headers = {"Content-type": "text/plain"}
for current_try in range(number_of_retries):
for current_try in range(self.number_of_retries):
should_retry = False
try:
response = self.requests_session.post(
@ -116,7 +122,7 @@ class LogzioSender:
timeout=self.network_timeout)
if response.status_code != 200:
if response.status_code == 400:
self.logger.info(
self.stdout_logger.info(
'Got 400 code from Logz.io. This means that '
'some of your logs are too big, or badly '
'formatted. response: %s', response.text)
@ -124,42 +130,41 @@ class LogzioSender:
break
if response.status_code == 401:
self.logger.info(
self.stdout_logger.info(
'You are not authorized with Logz.io! Token '
'OK? dropping logs...')
should_backup_to_disk = False
break
else:
self.logger.info(
self.stdout_logger.info(
'Got %s while sending logs to Logz.io, '
'Try (%s/%s). Response: %s',
response.status_code,
current_try + 1,
number_of_retries,
self.number_of_retries,
response.text)
should_retry = True
else:
self.logger.debug(
self.stdout_logger.debug(
'Successfully sent bulk of %s logs to '
'Logz.io!', len(logs_list))
should_backup_to_disk = False
break
except Exception as e:
self.logger.warning(
self.stdout_logger.warning(
'Got exception while sending logs to Logz.io, '
'Try (%s/%s). Message: %s',
current_try + 1, number_of_retries, e)
current_try + 1, self.number_of_retries, e)
should_retry = True
if should_retry:
sleep(sleep_between_retries)
sleep_between_retries *= 2
if should_backup_to_disk and self.backup_logs:
# Write to file
self.logger.error(
self.stdout_logger.error(
'Could not send logs to Logz.io after %s tries, '
'backing up to local file system', number_of_retries)
'backing up to local file system', self.number_of_retries)
backup_logs(logs_list, self.logger)
del logs_list


+ 1
- 1
requirements.txt View File

@ -1 +1 @@
requests>=2.23.0
requests>=2.27.0

+ 4
- 3
setup.py View File

@ -7,12 +7,13 @@ setup(
description="Logging handler to send logs to your Logz.io account with bulk SSL",
keywords="logging handler logz.io bulk https",
author="roiravhon",
author_email="roi@logz.io",
maintainer="tamir-michaeli",
mail="tamir.michaeli@logz.io",
url="https://github.com/logzio/logzio-python-handler/",
license="Apache License 2",
packages=find_packages(),
install_requires=[
"requests>=2.23.0"
"requests>=2.27.0"
],
test_requires=[
"future"
@ -21,6 +22,6 @@ setup(
classifiers=[
'Development Status :: 5 - Production/Stable',
'Intended Audience :: Developers',
'Programming Language :: Python :: 2.7'
'Programming Language :: Python :: 3.9'
]
)

+ 11
- 8
tests/test_logzioHandler.py View File

@ -15,7 +15,8 @@ class TestLogzioHandler(TestCase):
def test_json(self):
formatter = logging.Formatter(
'{ "appname":"%(name)s", "functionName":"%(funcName)s", \"lineNo":"%(lineno)d", "severity":"%(levelname)s", "message":"%(message)s"}')
'{ "appname":"%(name)s", "functionName":"%(funcName)s", \"lineNo":"%(lineno)d", "severity":"%('
'levelname)s", "message":"%(message)s"}')
self.handler.setFormatter(formatter)
record = logging.LogRecord(
@ -74,7 +75,7 @@ class TestLogzioHandler(TestCase):
'message': 'this is a test: moo.',
'path_name': 'handler_test.py',
'type': 'python'
}
}
)
def test_extra_formatting(self):
@ -105,7 +106,7 @@ class TestLogzioHandler(TestCase):
'path_name': 'handler_test.py',
'type': 'python',
'extra_key': 'extra_value'
}
}
)
def test_format_string_message(self):
@ -133,7 +134,7 @@ class TestLogzioHandler(TestCase):
'message': 'this is a test: moo.',
'path_name': 'handler_test.py',
'type': 'python'
}
}
)
def test_exc(self):
@ -147,7 +148,7 @@ class TestLogzioHandler(TestCase):
level=0,
pathname='handler_test.py',
lineno=10,
msg="this is a test: moo.",
msg='exception test:',
args=[],
exc_info=exc_info,
func='test_json'
@ -158,6 +159,7 @@ class TestLogzioHandler(TestCase):
formatted_message["exception"] = formatted_message["exception"].replace(os.path.abspath(__file__), "")
formatted_message["exception"] = re.sub(r", line \d+", "", formatted_message["exception"])
formatted_message["message"] = formatted_message["message"].replace(os.path.abspath(__file__), "")
self.assertDictEqual(
{
@ -165,11 +167,12 @@ class TestLogzioHandler(TestCase):
'line_number': 10,
'log_level': 'NOTSET',
'logger': 'my-logger',
'message': 'this is a test: moo.',
'exception': 'Traceback (most recent call last):\n\n File "", in test_exc\n raise ValueError("oops.")\n\nValueError: oops.\n',
'message': f'exception test:\nTraceback (most recent call last):\n File "", line 142, in test_exc\n raise '
'ValueError("oops.")\nValueError: oops.',
'exception': 'Traceback (most recent call last):\n\n File "", in test_exc\n raise ValueError('
'"oops.")\n\nValueError: oops.\n',
'path_name': 'handler_test.py',
'type': 'python'
},
formatted_message
)

+ 8
- 4
tests/test_logzioSender.py View File

@ -24,6 +24,8 @@ class TestLogzioSender(TestCase):
self.logzio_listener.clear_logs_buffer()
self.logzio_listener.clear_server_error()
self.logs_drain_timeout = 1
self.retries_no = 4
self.retry_timeout = 2
logging_configuration = {
"version": 1,
@ -42,7 +44,9 @@ class TestLogzioSender(TestCase):
'logzio_type': "type",
'logs_drain_timeout': self.logs_drain_timeout,
'url': "http://" + self.logzio_listener.get_host() + ":" + str(self.logzio_listener.get_port()),
'debug': True
'debug': True,
'retries_no': self.retries_no,
'retry_timeout': self.retry_timeout
}
},
"loggers": {
@ -88,7 +92,7 @@ class TestLogzioSender(TestCase):
self.logzio_listener.clear_server_error()
time.sleep(self.logs_drain_timeout * 2 * 4) # Longer, because of the retry
time.sleep(self.logs_drain_timeout * self.retry_timeout * self.retries_no) # Longer, because of the retry
self.assertTrue(self.logzio_listener.find_log(log_message))
@ -100,7 +104,7 @@ class TestLogzioSender(TestCase):
# Make sure no file is present
self.assertEqual(len(_find("logzio-failures-*.txt", ".")), 0)
time.sleep(2 * 2 * 2 * 2 * 2) # All of the retries
time.sleep(self.retries_no*self.retry_timeout*2*2) # All of the retries
failure_files = _find("logzio-failures-*.txt", ".")
self.assertEqual(len(failure_files), 1)
@ -118,7 +122,7 @@ class TestLogzioSender(TestCase):
# Make sure no file is present
self.assertEqual(len(_find("logzio-failures-*.txt", ".")), 0)
time.sleep(2 * 2 * 2 * 2 * 2) # All of the retries
time.sleep(self.retries_no*self.retry_timeout) # All of the retries
# Make sure no file was created
self.assertEqual(len(_find("logzio-failures-*.txt", ".")), 0)


Write Preview
Loading…
Cancel
Save