diff --git a/src/pylorax/__init__.py b/src/pylorax/__init__.py index f420d68f5..7aecaec6b 100644 --- a/src/pylorax/__init__.py +++ b/src/pylorax/__init__.py @@ -168,7 +168,7 @@ def init_stream_logging(self): logger.addHandler(sh) def init_file_logging(self, logdir, logname="pylorax.log"): - fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w") + fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w", encoding="UTF-8") fh.setLevel(logging.DEBUG) logger.addHandler(fh) @@ -411,7 +411,7 @@ def setup_logging(logfile, theLogger): logger.addHandler(sh) theLogger.addHandler(sh) - fh = logging.FileHandler(filename=logfile, mode="w") + fh = logging.FileHandler(filename=logfile, mode="w", encoding="UTF-8") fh.setLevel(logging.DEBUG) fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") fh.setFormatter(fmt) @@ -421,7 +421,7 @@ def setup_logging(logfile, theLogger): # External program output log program_log.setLevel(logging.DEBUG) f = os.path.abspath(os.path.dirname(logfile))+"/program.log" - fh = logging.FileHandler(filename=f, mode="w") + fh = logging.FileHandler(filename=f, mode="w", encoding="UTF-8") fh.setLevel(logging.DEBUG) fmt = logging.Formatter("%(asctime)s %(levelname)s: %(message)s") fh.setFormatter(fmt) diff --git a/src/pylorax/executils.py b/src/pylorax/executils.py index ffb26b68a..5981c5213 100644 --- a/src/pylorax/executils.py +++ b/src/pylorax/executils.py @@ -127,7 +127,7 @@ def preexec(): preexec_fn=preexec, cwd=root, env=env, **kwargs) def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_output=True, - binary_output=False, filter_stderr=False, raise_err=False, callback=None, + filter_stderr=False, raise_err=False, callback=None, env_add=None, reset_handlers=True, reset_lang=True): """ Run an external program, log the output and return it to the caller @@ -137,7 +137,6 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou :param stdout: Optional file object to write the output to. :param env_prune: environment variable to remove before execution :param log_output: whether to log the output of command - :param binary_output: whether to treat the output of command as binary data :param filter_stderr: whether to exclude the contents of stderr from the returned output :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero :param callback: method to call while waiting for process to finish, passed Popen object @@ -154,7 +153,7 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou stderr = subprocess.STDOUT proc = startProgram(argv, root=root, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr, - env_prune=env_prune, universal_newlines=not binary_output, + env_prune=env_prune, text=True, encoding="UTF-8", env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang) output_string = None @@ -169,12 +168,9 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou else: (output_string, err_string) = proc.communicate() if output_string: - if binary_output: - output_lines = [output_string] - else: - if output_string[-1] != "\n": - output_string = output_string + "\n" - output_lines = output_string.splitlines(True) + if output_string[-1] != "\n": + output_string = output_string + "\n" + output_lines = output_string.splitlines(True) if log_output: with program_log_lock: @@ -207,7 +203,7 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou return (proc.returncode, output_string) def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune=None, - log_output=True, binary_output=False, raise_err=False, callback=None, + log_output=True, raise_err=False, callback=None, env_add=None, reset_handlers=True, reset_lang=True): """ Run an external program and redirect the output to a file. @@ -218,7 +214,6 @@ def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune :param root: The directory to chroot to before running command. :param env_prune: environment variable to remove before execution :param log_output: whether to log the output of command - :param binary_output: whether to treat the output of command as binary data :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero :param callback: method to call while waiting for process to finish, passed Popen object :param env_add: environment variables to add before execution @@ -228,7 +223,7 @@ def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune """ argv = [command] + list(argv) return _run_program(argv, stdin=stdin, stdout=stdout, root=root, env_prune=env_prune, - log_output=log_output, binary_output=binary_output, raise_err=raise_err, callback=callback, + log_output=log_output, raise_err=raise_err, callback=callback, env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang)[0] def execWithCapture(command, argv, stdin=None, root='/', log_output=True, filter_stderr=False, @@ -290,7 +285,7 @@ def __init__(self, proc, argv, callback): self._proc = proc self._argv = argv self._callback = callback - self._data = "" + self._data = b"" def __iter__(self): return self @@ -311,12 +306,12 @@ def __next__(self): if select.select([self._proc.stdout], [], [], 0)[0]: size = len(self._proc.stdout.peek(1)) if size > 0: - self._data += self._proc.stdout.read(size).decode("utf-8") + self._data += self._proc.stdout.read(size) - if self._data.find("\n") >= 0: - line = self._data.split("\n", 1) + if self._data.find(b"\n") >= 0: + line = self._data.split(b"\n", 1) self._data = line[1] - return line[0] + return line[0].decode("utf-8", "ignore") if self._proc.poll() is not None or not self._callback(self._proc): # Output finished, wait 60s for the process to end diff --git a/tests/pylorax/test_executils.py b/tests/pylorax/test_executils.py index 13c773838..60670bf98 100644 --- a/tests/pylorax/test_executils.py +++ b/tests/pylorax/test_executils.py @@ -57,7 +57,7 @@ def test_execWithRedirect(self): program_log.setLevel(logging.INFO) tmp_f = tempfile.NamedTemporaryFile(prefix="lorax.test.log.", delete=False) - fh = logging.FileHandler(filename=tmp_f.name, mode="w") + fh = logging.FileHandler(filename=tmp_f.name, mode="w", encoding="UTF-8") program_log.addHandler(fh) try: @@ -66,7 +66,7 @@ def test_execWithRedirect(self): self.assertEqual(rc, 1) fh.close() - with open(tmp_f.name, "r") as f: + with open(tmp_f.name, "r", encoding="UTF-8") as f: logged_text = f.readlines()[-1].strip() self.assertEqual(logged_text, "The Once-ler was here.") finally: