From a7eeca0a23caa81d8583de08f228ea9a0d00fd81 Mon Sep 17 00:00:00 2001 From: Christian Herdtweck Date: Mon, 15 Oct 2018 10:31:26 +0200 Subject: [PATCH] Move log_helper from "utils" to "common" --- oletools/common/log_helper/__init__.py | 5 +++++ oletools/common/log_helper/_json_formatter.py | 24 ++++++++++++++++++++++++ oletools/common/log_helper/_logger_adapter.py | 30 ++++++++++++++++++++++++++++++ oletools/common/log_helper/_root_logger_wrapper.py | 24 ++++++++++++++++++++++++ oletools/common/log_helper/log_helper.py | 194 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ oletools/util/__init__.py | 0 oletools/util/log_helper/__init__.py | 5 ----- oletools/util/log_helper/_json_formatter.py | 24 ------------------------ oletools/util/log_helper/_logger_adapter.py | 30 ------------------------------ oletools/util/log_helper/_root_logger_wrapper.py | 24 ------------------------ oletools/util/log_helper/log_helper.py | 194 -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 11 files changed, 277 insertions(+), 277 deletions(-) create mode 100644 oletools/common/log_helper/__init__.py create mode 100644 oletools/common/log_helper/_json_formatter.py create mode 100644 oletools/common/log_helper/_logger_adapter.py create mode 100644 oletools/common/log_helper/_root_logger_wrapper.py create mode 100644 oletools/common/log_helper/log_helper.py delete mode 100644 oletools/util/__init__.py delete mode 100644 oletools/util/log_helper/__init__.py delete mode 100644 oletools/util/log_helper/_json_formatter.py delete mode 100644 oletools/util/log_helper/_logger_adapter.py delete mode 100644 oletools/util/log_helper/_root_logger_wrapper.py delete mode 100644 oletools/util/log_helper/log_helper.py diff --git a/oletools/common/log_helper/__init__.py b/oletools/common/log_helper/__init__.py new file mode 100644 index 0000000..7a027c2 --- /dev/null +++ b/oletools/common/log_helper/__init__.py @@ -0,0 +1,5 @@ +from . import log_helper as log_helper_ + +log_helper = log_helper_.LogHelper() + +__all__ = ['log_helper'] diff --git a/oletools/common/log_helper/_json_formatter.py b/oletools/common/log_helper/_json_formatter.py new file mode 100644 index 0000000..4c5e337 --- /dev/null +++ b/oletools/common/log_helper/_json_formatter.py @@ -0,0 +1,24 @@ +import logging +import json + + +class JsonFormatter(logging.Formatter): + """ + Format every message to be logged as a JSON object + """ + _is_first_line = True + + def format(self, record): + """ + Since we don't buffer messages, we always prepend messages with a comma to make + the output JSON-compatible. The only exception is when printing the first line, + so we need to keep track of it. + """ + json_dict = dict(msg=record.msg, level=record.levelname) + formatted_message = ' ' + json.dumps(json_dict) + + if self._is_first_line: + self._is_first_line = False + return formatted_message + + return ', ' + formatted_message diff --git a/oletools/common/log_helper/_logger_adapter.py b/oletools/common/log_helper/_logger_adapter.py new file mode 100644 index 0000000..75e331a --- /dev/null +++ b/oletools/common/log_helper/_logger_adapter.py @@ -0,0 +1,30 @@ +import logging +from . import _root_logger_wrapper + + +class OletoolsLoggerAdapter(logging.LoggerAdapter): + """ + Adapter class for all loggers returned by the logging module. + """ + _json_enabled = None + + def print_str(self, message): + """ + This function replaces normal print() calls so we can format them as JSON + when needed or just print them right away otherwise. + """ + if self._json_enabled and self._json_enabled(): + # Messages from this function should always be printed, + # so when using JSON we log using the same level that set + self.log(_root_logger_wrapper.level(), message) + else: + print(message) + + def set_json_enabled_function(self, json_enabled): + """ + Set a function to be called to check whether JSON output is enabled. + """ + self._json_enabled = json_enabled + + def level(self): + return self.logger.level diff --git a/oletools/common/log_helper/_root_logger_wrapper.py b/oletools/common/log_helper/_root_logger_wrapper.py new file mode 100644 index 0000000..273d5c6 --- /dev/null +++ b/oletools/common/log_helper/_root_logger_wrapper.py @@ -0,0 +1,24 @@ +import logging + + +def is_logging_initialized(): + """ + We use the same strategy as the logging module when checking if + the logging was initialized - look for handlers in the root logger + """ + return len(logging.root.handlers) > 0 + + +def set_formatter(fmt): + """ + Set the formatter to be used by every handler of the root logger. + """ + if not is_logging_initialized(): + return + + for handler in logging.root.handlers: + handler.setFormatter(fmt) + + +def level(): + return logging.root.level diff --git a/oletools/common/log_helper/log_helper.py b/oletools/common/log_helper/log_helper.py new file mode 100644 index 0000000..7a7fb02 --- /dev/null +++ b/oletools/common/log_helper/log_helper.py @@ -0,0 +1,194 @@ +""" +log_helper.py + +General logging helpers + +.. codeauthor:: Intra2net AG +""" + +# === LICENSE ================================================================= +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# ----------------------------------------------------------------------------- +# CHANGELOG: +# 2017-12-07 v0.01 CH: - first version +# 2018-02-05 v0.02 SA: - fixed log level selection and reformatted code +# 2018-02-06 v0.03 SA: - refactored code to deal with NullHandlers +# 2018-02-07 v0.04 SA: - fixed control of handlers propagation +# 2018-04-23 v0.05 SA: - refactored the whole logger to use an OOP approach + +# ----------------------------------------------------------------------------- +# TODO: + + +from ._json_formatter import JsonFormatter +from ._logger_adapter import OletoolsLoggerAdapter +from . import _root_logger_wrapper +import logging +import sys + + +LOG_LEVELS = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL +} + +DEFAULT_LOGGER_NAME = 'oletools' +DEFAULT_MESSAGE_FORMAT = '%(levelname)-8s %(message)s' + + +class LogHelper: + def __init__(self): + self._all_names = set() # set so we do not have duplicates + self._use_json = False + self._is_enabled = False + + def get_or_create_silent_logger(self, name=DEFAULT_LOGGER_NAME, level=logging.CRITICAL + 1): + """ + Get a logger or create one if it doesn't exist, setting a NullHandler + as the handler (to avoid printing to the console). + By default we also use a higher logging level so every message will + be ignored. + This will prevent oletools from logging unnecessarily when being imported + from external tools. + """ + return self._get_or_create_logger(name, level, logging.NullHandler()) + + def enable_logging(self, use_json, level, log_format=DEFAULT_MESSAGE_FORMAT, stream=None): + """ + This function initializes the root logger and enables logging. + We set the level of the root logger to the one passed by calling logging.basicConfig. + We also set the level of every logger we created to 0 (logging.NOTSET), meaning that + the level of the root logger will be used to tell if messages should be logged. + Additionally, since our loggers use the NullHandler, they won't log anything themselves, + but due to having propagation enabled they will pass messages to the root logger, + which in turn will log to the stream set in this function. + Since the root logger is the one doing the work, when using JSON we set its formatter + so that every message logged is JSON-compatible. + """ + if self._is_enabled: + raise ValueError('re-enabling logging. Not sure whether that is ok...') + + log_level = LOG_LEVELS[level] + logging.basicConfig(level=log_level, format=log_format, stream=stream) + self._is_enabled = True + + self._use_json = use_json + sys.excepthook = self._get_except_hook(sys.excepthook) + + # since there could be loggers already created we go through all of them + # and set their levels to 0 so they will use the root logger's level + for name in self._all_names: + logger = self.get_or_create_silent_logger(name) + self._set_logger_level(logger, logging.NOTSET) + + # add a JSON formatter to the root logger, which will be used by every logger + if self._use_json: + _root_logger_wrapper.set_formatter(JsonFormatter()) + print('[') + + def end_logging(self): + """ + Must be called at the end of the main function if the caller wants + json-compatible output + """ + if not self._is_enabled: + return + self._is_enabled = False + + # end logging + self._all_names = set() + logging.shutdown() + + # end json list + if self._use_json: + print(']') + self._use_json = False + + def _get_except_hook(self, old_hook): + """ + Global hook for exceptions so we can always end logging. + We wrap any hook currently set to avoid overwriting global hooks set by oletools. + Note that this is only called by enable_logging, which in turn is called by + the main() function in oletools' scripts. When scripts are being imported this + code won't execute and won't affect global hooks. + """ + def hook(exctype, value, traceback): + self.end_logging() + old_hook(exctype, value, traceback) + + return hook + + def _get_or_create_logger(self, name, level, handler=None): + """ + Get or create a new logger. This newly created logger will have the + handler and level that was passed, but if it already exists it's not changed. + We also wrap the logger in an adapter so we can easily extend its functionality. + """ + + # logging.getLogger creates a logger if it doesn't exist, + # so we need to check before calling it + if handler and not self._log_exists(name): + logger = logging.getLogger(name) + logger.addHandler(handler) + self._set_logger_level(logger, level) + else: + logger = logging.getLogger(name) + + # Keep track of every logger we created so we can easily change + # their levels whenever needed + self._all_names.add(name) + + adapted_logger = OletoolsLoggerAdapter(logger, None) + adapted_logger.set_json_enabled_function(lambda: self._use_json) + + return adapted_logger + + @staticmethod + def _set_logger_level(logger, level): + """ + If the logging is already initialized, we set the level of our logger + to 0, meaning that it will reuse the level of the root logger. + That means that if the root logger level changes, we will keep using + its level and not logging unnecessarily. + """ + + # if this log was wrapped, unwrap it to set the level + if isinstance(logger, OletoolsLoggerAdapter): + logger = logger.logger + + if _root_logger_wrapper.is_logging_initialized(): + logger.setLevel(logging.NOTSET) + else: + logger.setLevel(level) + + @staticmethod + def _log_exists(name): + """ + We check the log manager instead of our global _all_names variable + since the logger could have been created outside of the helper + """ + return name in logging.Logger.manager.loggerDict diff --git a/oletools/util/__init__.py b/oletools/util/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/oletools/util/__init__.py +++ /dev/null diff --git a/oletools/util/log_helper/__init__.py b/oletools/util/log_helper/__init__.py deleted file mode 100644 index 7a027c2..0000000 --- a/oletools/util/log_helper/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from . import log_helper as log_helper_ - -log_helper = log_helper_.LogHelper() - -__all__ = ['log_helper'] diff --git a/oletools/util/log_helper/_json_formatter.py b/oletools/util/log_helper/_json_formatter.py deleted file mode 100644 index 4c5e337..0000000 --- a/oletools/util/log_helper/_json_formatter.py +++ /dev/null @@ -1,24 +0,0 @@ -import logging -import json - - -class JsonFormatter(logging.Formatter): - """ - Format every message to be logged as a JSON object - """ - _is_first_line = True - - def format(self, record): - """ - Since we don't buffer messages, we always prepend messages with a comma to make - the output JSON-compatible. The only exception is when printing the first line, - so we need to keep track of it. - """ - json_dict = dict(msg=record.msg, level=record.levelname) - formatted_message = ' ' + json.dumps(json_dict) - - if self._is_first_line: - self._is_first_line = False - return formatted_message - - return ', ' + formatted_message diff --git a/oletools/util/log_helper/_logger_adapter.py b/oletools/util/log_helper/_logger_adapter.py deleted file mode 100644 index 75e331a..0000000 --- a/oletools/util/log_helper/_logger_adapter.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging -from . import _root_logger_wrapper - - -class OletoolsLoggerAdapter(logging.LoggerAdapter): - """ - Adapter class for all loggers returned by the logging module. - """ - _json_enabled = None - - def print_str(self, message): - """ - This function replaces normal print() calls so we can format them as JSON - when needed or just print them right away otherwise. - """ - if self._json_enabled and self._json_enabled(): - # Messages from this function should always be printed, - # so when using JSON we log using the same level that set - self.log(_root_logger_wrapper.level(), message) - else: - print(message) - - def set_json_enabled_function(self, json_enabled): - """ - Set a function to be called to check whether JSON output is enabled. - """ - self._json_enabled = json_enabled - - def level(self): - return self.logger.level diff --git a/oletools/util/log_helper/_root_logger_wrapper.py b/oletools/util/log_helper/_root_logger_wrapper.py deleted file mode 100644 index 273d5c6..0000000 --- a/oletools/util/log_helper/_root_logger_wrapper.py +++ /dev/null @@ -1,24 +0,0 @@ -import logging - - -def is_logging_initialized(): - """ - We use the same strategy as the logging module when checking if - the logging was initialized - look for handlers in the root logger - """ - return len(logging.root.handlers) > 0 - - -def set_formatter(fmt): - """ - Set the formatter to be used by every handler of the root logger. - """ - if not is_logging_initialized(): - return - - for handler in logging.root.handlers: - handler.setFormatter(fmt) - - -def level(): - return logging.root.level diff --git a/oletools/util/log_helper/log_helper.py b/oletools/util/log_helper/log_helper.py deleted file mode 100644 index 7a7fb02..0000000 --- a/oletools/util/log_helper/log_helper.py +++ /dev/null @@ -1,194 +0,0 @@ -""" -log_helper.py - -General logging helpers - -.. codeauthor:: Intra2net AG -""" - -# === LICENSE ================================================================= -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -# ----------------------------------------------------------------------------- -# CHANGELOG: -# 2017-12-07 v0.01 CH: - first version -# 2018-02-05 v0.02 SA: - fixed log level selection and reformatted code -# 2018-02-06 v0.03 SA: - refactored code to deal with NullHandlers -# 2018-02-07 v0.04 SA: - fixed control of handlers propagation -# 2018-04-23 v0.05 SA: - refactored the whole logger to use an OOP approach - -# ----------------------------------------------------------------------------- -# TODO: - - -from ._json_formatter import JsonFormatter -from ._logger_adapter import OletoolsLoggerAdapter -from . import _root_logger_wrapper -import logging -import sys - - -LOG_LEVELS = { - 'debug': logging.DEBUG, - 'info': logging.INFO, - 'warning': logging.WARNING, - 'error': logging.ERROR, - 'critical': logging.CRITICAL -} - -DEFAULT_LOGGER_NAME = 'oletools' -DEFAULT_MESSAGE_FORMAT = '%(levelname)-8s %(message)s' - - -class LogHelper: - def __init__(self): - self._all_names = set() # set so we do not have duplicates - self._use_json = False - self._is_enabled = False - - def get_or_create_silent_logger(self, name=DEFAULT_LOGGER_NAME, level=logging.CRITICAL + 1): - """ - Get a logger or create one if it doesn't exist, setting a NullHandler - as the handler (to avoid printing to the console). - By default we also use a higher logging level so every message will - be ignored. - This will prevent oletools from logging unnecessarily when being imported - from external tools. - """ - return self._get_or_create_logger(name, level, logging.NullHandler()) - - def enable_logging(self, use_json, level, log_format=DEFAULT_MESSAGE_FORMAT, stream=None): - """ - This function initializes the root logger and enables logging. - We set the level of the root logger to the one passed by calling logging.basicConfig. - We also set the level of every logger we created to 0 (logging.NOTSET), meaning that - the level of the root logger will be used to tell if messages should be logged. - Additionally, since our loggers use the NullHandler, they won't log anything themselves, - but due to having propagation enabled they will pass messages to the root logger, - which in turn will log to the stream set in this function. - Since the root logger is the one doing the work, when using JSON we set its formatter - so that every message logged is JSON-compatible. - """ - if self._is_enabled: - raise ValueError('re-enabling logging. Not sure whether that is ok...') - - log_level = LOG_LEVELS[level] - logging.basicConfig(level=log_level, format=log_format, stream=stream) - self._is_enabled = True - - self._use_json = use_json - sys.excepthook = self._get_except_hook(sys.excepthook) - - # since there could be loggers already created we go through all of them - # and set their levels to 0 so they will use the root logger's level - for name in self._all_names: - logger = self.get_or_create_silent_logger(name) - self._set_logger_level(logger, logging.NOTSET) - - # add a JSON formatter to the root logger, which will be used by every logger - if self._use_json: - _root_logger_wrapper.set_formatter(JsonFormatter()) - print('[') - - def end_logging(self): - """ - Must be called at the end of the main function if the caller wants - json-compatible output - """ - if not self._is_enabled: - return - self._is_enabled = False - - # end logging - self._all_names = set() - logging.shutdown() - - # end json list - if self._use_json: - print(']') - self._use_json = False - - def _get_except_hook(self, old_hook): - """ - Global hook for exceptions so we can always end logging. - We wrap any hook currently set to avoid overwriting global hooks set by oletools. - Note that this is only called by enable_logging, which in turn is called by - the main() function in oletools' scripts. When scripts are being imported this - code won't execute and won't affect global hooks. - """ - def hook(exctype, value, traceback): - self.end_logging() - old_hook(exctype, value, traceback) - - return hook - - def _get_or_create_logger(self, name, level, handler=None): - """ - Get or create a new logger. This newly created logger will have the - handler and level that was passed, but if it already exists it's not changed. - We also wrap the logger in an adapter so we can easily extend its functionality. - """ - - # logging.getLogger creates a logger if it doesn't exist, - # so we need to check before calling it - if handler and not self._log_exists(name): - logger = logging.getLogger(name) - logger.addHandler(handler) - self._set_logger_level(logger, level) - else: - logger = logging.getLogger(name) - - # Keep track of every logger we created so we can easily change - # their levels whenever needed - self._all_names.add(name) - - adapted_logger = OletoolsLoggerAdapter(logger, None) - adapted_logger.set_json_enabled_function(lambda: self._use_json) - - return adapted_logger - - @staticmethod - def _set_logger_level(logger, level): - """ - If the logging is already initialized, we set the level of our logger - to 0, meaning that it will reuse the level of the root logger. - That means that if the root logger level changes, we will keep using - its level and not logging unnecessarily. - """ - - # if this log was wrapped, unwrap it to set the level - if isinstance(logger, OletoolsLoggerAdapter): - logger = logger.logger - - if _root_logger_wrapper.is_logging_initialized(): - logger.setLevel(logging.NOTSET) - else: - logger.setLevel(level) - - @staticmethod - def _log_exists(name): - """ - We check the log manager instead of our global _all_names variable - since the logger could have been created outside of the helper - """ - return name in logging.Logger.manager.loggerDict -- libgit2 0.21.4