From aeb10148a3a8b054bca12a369c59c7f9d9fbe95d Mon Sep 17 00:00:00 2001 From: Oleg Sviridov Date: Sat, 13 Sep 2025 15:07:57 +0300 Subject: [PATCH 1/5] executils: decoding after deserialization To prevent an error like: "ERROR livemedia-creator: 'utf-8' codec can't decode byte 0xd1 in position 4095: unexpected end of data", when using a ru-RU.UTF8 locale. It is proposed to decode already deserialized data. (cherry picked from commit 28f583659a23484f5718b1055cc8b9e5787be748) Related: RHEL-121651 --- src/pylorax/executils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pylorax/executils.py b/src/pylorax/executils.py index ffb26b68a..538b7a9b6 100644 --- a/src/pylorax/executils.py +++ b/src/pylorax/executils.py @@ -290,7 +290,7 @@ def __init__(self, proc, argv, callback): self._proc = proc self._argv = argv self._callback = callback - self._data = "" + self._data = b"" def __iter__(self): return self @@ -311,12 +311,12 @@ def __next__(self): if select.select([self._proc.stdout], [], [], 0)[0]: size = len(self._proc.stdout.peek(1)) if size > 0: - self._data += self._proc.stdout.read(size).decode("utf-8") + self._data += self._proc.stdout.read(size) - if self._data.find("\n") >= 0: - line = self._data.split("\n", 1) + if self._data.find(b"\n") >= 0: + line = self._data.split(b"\n", 1) self._data = line[1] - return line[0] + return line[0].decode("utf-8") if self._proc.poll() is not None or not self._callback(self._proc): # Output finished, wait 60s for the process to end From 993fa0546e2f40fcb2486713847985756fa12167 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Tue, 12 Aug 2025 11:58:32 -0700 Subject: [PATCH 2/5] executils: Set encoding to UTF-8 for _run_program This allows it to properly handle unicode output from the process that is being run. I have been able to reproduce the failure by running lorax in a VM with a modified runtime-postinstall that calls a python program that outputs unicode, or with glib2-2.85.3-1.fc43 -- but I am unable to cause it to fail in the test suite. (cherry picked from commit 3a5327868b201f2ab41fbf10e11b1d2633eb1b61) Related: RHEL-121651 --- src/pylorax/executils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pylorax/executils.py b/src/pylorax/executils.py index 538b7a9b6..440b02fa3 100644 --- a/src/pylorax/executils.py +++ b/src/pylorax/executils.py @@ -155,6 +155,7 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou proc = startProgram(argv, root=root, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr, env_prune=env_prune, universal_newlines=not binary_output, + encoding="UTF-8", env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang) output_string = None From 114c47be5c3da36ca1ab5fe51ee68fef9852fa99 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Tue, 12 Aug 2025 13:31:18 -0700 Subject: [PATCH 3/5] logging: Set encoding=UTF-8 on FileHandler This allows unicode to be passed to the logfiles without raising an error. (cherry picked from commit 24d17f81faec374c4e5ce84590938fe159445a6f) Related: RHEL-121651 --- src/pylorax/__init__.py | 6 +++--- tests/pylorax/test_executils.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pylorax/__init__.py b/src/pylorax/__init__.py index f420d68f5..7aecaec6b 100644 --- a/src/pylorax/__init__.py +++ b/src/pylorax/__init__.py @@ -168,7 +168,7 @@ def init_stream_logging(self): logger.addHandler(sh) def init_file_logging(self, logdir, logname="pylorax.log"): - fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w") + fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w", encoding="UTF-8") fh.setLevel(logging.DEBUG) logger.addHandler(fh) @@ -411,7 +411,7 @@ def setup_logging(logfile, theLogger): logger.addHandler(sh) theLogger.addHandler(sh) - fh = logging.FileHandler(filename=logfile, mode="w") + fh = logging.FileHandler(filename=logfile, mode="w", encoding="UTF-8") fh.setLevel(logging.DEBUG) fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") fh.setFormatter(fmt) @@ -421,7 +421,7 @@ def setup_logging(logfile, theLogger): # External program output log program_log.setLevel(logging.DEBUG) f = os.path.abspath(os.path.dirname(logfile))+"/program.log" - fh = logging.FileHandler(filename=f, mode="w") + fh = logging.FileHandler(filename=f, mode="w", encoding="UTF-8") fh.setLevel(logging.DEBUG) fmt = logging.Formatter("%(asctime)s %(levelname)s: %(message)s") fh.setFormatter(fmt) diff --git a/tests/pylorax/test_executils.py b/tests/pylorax/test_executils.py index 13c773838..60670bf98 100644 --- a/tests/pylorax/test_executils.py +++ b/tests/pylorax/test_executils.py @@ -57,7 +57,7 @@ def test_execWithRedirect(self): program_log.setLevel(logging.INFO) tmp_f = tempfile.NamedTemporaryFile(prefix="lorax.test.log.", delete=False) - fh = logging.FileHandler(filename=tmp_f.name, mode="w") + fh = logging.FileHandler(filename=tmp_f.name, mode="w", encoding="UTF-8") program_log.addHandler(fh) try: @@ -66,7 +66,7 @@ def test_execWithRedirect(self): self.assertEqual(rc, 1) fh.close() - with open(tmp_f.name, "r") as f: + with open(tmp_f.name, "r", encoding="UTF-8") as f: logged_text = f.readlines()[-1].strip() self.assertEqual(logged_text, "The Once-ler was here.") finally: From 8f1c069944197d9fddd7e43bf650a080dfb50bc9 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Wed, 13 Aug 2025 09:46:56 -0700 Subject: [PATCH 4/5] executils: Remove binary_output flag This was never used in lorax, is always set to false, and now with encoding set to UTF8 for these functions it wouldn't be binary anyway. Also sets text=True in the _run_program call to startProgram, previously the old universal_newlines was set from the state of binary_output. (cherry picked from commit 42786c2abdf7e7b9294c8727db2d3dbadfd5f153) Related: RHEL-121651 --- src/pylorax/executils.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/pylorax/executils.py b/src/pylorax/executils.py index 440b02fa3..ba2b16ca2 100644 --- a/src/pylorax/executils.py +++ b/src/pylorax/executils.py @@ -127,7 +127,7 @@ def preexec(): preexec_fn=preexec, cwd=root, env=env, **kwargs) def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_output=True, - binary_output=False, filter_stderr=False, raise_err=False, callback=None, + filter_stderr=False, raise_err=False, callback=None, env_add=None, reset_handlers=True, reset_lang=True): """ Run an external program, log the output and return it to the caller @@ -137,7 +137,6 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou :param stdout: Optional file object to write the output to. :param env_prune: environment variable to remove before execution :param log_output: whether to log the output of command - :param binary_output: whether to treat the output of command as binary data :param filter_stderr: whether to exclude the contents of stderr from the returned output :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero :param callback: method to call while waiting for process to finish, passed Popen object @@ -154,8 +153,7 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou stderr = subprocess.STDOUT proc = startProgram(argv, root=root, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr, - env_prune=env_prune, universal_newlines=not binary_output, - encoding="UTF-8", + env_prune=env_prune, text=True, encoding="UTF-8", env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang) output_string = None @@ -170,12 +168,9 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou else: (output_string, err_string) = proc.communicate() if output_string: - if binary_output: - output_lines = [output_string] - else: - if output_string[-1] != "\n": - output_string = output_string + "\n" - output_lines = output_string.splitlines(True) + if output_string[-1] != "\n": + output_string = output_string + "\n" + output_lines = output_string.splitlines(True) if log_output: with program_log_lock: @@ -208,7 +203,7 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou return (proc.returncode, output_string) def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune=None, - log_output=True, binary_output=False, raise_err=False, callback=None, + log_output=True, raise_err=False, callback=None, env_add=None, reset_handlers=True, reset_lang=True): """ Run an external program and redirect the output to a file. @@ -219,7 +214,6 @@ def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune :param root: The directory to chroot to before running command. :param env_prune: environment variable to remove before execution :param log_output: whether to log the output of command - :param binary_output: whether to treat the output of command as binary data :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero :param callback: method to call while waiting for process to finish, passed Popen object :param env_add: environment variables to add before execution @@ -229,7 +223,7 @@ def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune """ argv = [command] + list(argv) return _run_program(argv, stdin=stdin, stdout=stdout, root=root, env_prune=env_prune, - log_output=log_output, binary_output=binary_output, raise_err=raise_err, callback=callback, + log_output=log_output, raise_err=raise_err, callback=callback, env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang)[0] def execWithCapture(command, argv, stdin=None, root='/', log_output=True, filter_stderr=False, From 34f15abedd96ad3eaee82c987beb8c075f549b78 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Mon, 2 Feb 2026 15:04:24 -0800 Subject: [PATCH 5/5] executils: Ignore utf-8 decode errors We have no control over the data stream so if what we receive from the program (anaconda, or whatever) is broken the best we can do is ignore errors and not crash because of a decode error. (cherry picked from commit 56a0c3af16f392fb8edfc33572d03fc5bcd85e3f) Resolves: RHEL-121651 --- src/pylorax/executils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pylorax/executils.py b/src/pylorax/executils.py index ba2b16ca2..5981c5213 100644 --- a/src/pylorax/executils.py +++ b/src/pylorax/executils.py @@ -311,7 +311,7 @@ def __next__(self): if self._data.find(b"\n") >= 0: line = self._data.split(b"\n", 1) self._data = line[1] - return line[0].decode("utf-8") + return line[0].decode("utf-8", "ignore") if self._proc.poll() is not None or not self._callback(self._proc): # Output finished, wait 60s for the process to end