Backport execReadlines UTF-8 fix

src/pylorax/__init__.py

@@ -168,7 +168,7 @@ def init_stream_logging(self):
         logger.addHandler(sh)
 
     def init_file_logging(self, logdir, logname="pylorax.log"):
-        fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w")
+        fh = logging.FileHandler(filename=joinpaths(logdir, logname), mode="w", encoding="UTF-8")
         fh.setLevel(logging.DEBUG)
         logger.addHandler(fh)
 
@@ -411,7 +411,7 @@ def setup_logging(logfile, theLogger):
     logger.addHandler(sh)
     theLogger.addHandler(sh)
 
-    fh = logging.FileHandler(filename=logfile, mode="w")
+    fh = logging.FileHandler(filename=logfile, mode="w", encoding="UTF-8")
     fh.setLevel(logging.DEBUG)
     fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
     fh.setFormatter(fmt)
@@ -421,7 +421,7 @@ def setup_logging(logfile, theLogger):
     # External program output log
     program_log.setLevel(logging.DEBUG)
     f = os.path.abspath(os.path.dirname(logfile))+"/program.log"
-    fh = logging.FileHandler(filename=f, mode="w")
+    fh = logging.FileHandler(filename=f, mode="w", encoding="UTF-8")
     fh.setLevel(logging.DEBUG)
     fmt = logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
     fh.setFormatter(fmt)

src/pylorax/executils.py

@@ -127,7 +127,7 @@ def preexec():
                             preexec_fn=preexec, cwd=root, env=env, **kwargs)
 
 def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_output=True,
-        binary_output=False, filter_stderr=False, raise_err=False, callback=None,
+        filter_stderr=False, raise_err=False, callback=None,
         env_add=None, reset_handlers=True, reset_lang=True):
     """ Run an external program, log the output and return it to the caller
 
@@ -137,7 +137,6 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou
         :param stdout: Optional file object to write the output to.
         :param env_prune: environment variable to remove before execution
         :param log_output: whether to log the output of command
-        :param binary_output: whether to treat the output of command as binary data
         :param filter_stderr: whether to exclude the contents of stderr from the returned output
         :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero
         :param callback: method to call while waiting for process to finish, passed Popen object
@@ -154,7 +153,7 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou
             stderr = subprocess.STDOUT
 
         proc = startProgram(argv, root=root, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr,
-                            env_prune=env_prune, universal_newlines=not binary_output,
+                            env_prune=env_prune, text=True, encoding="UTF-8",
                             env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang)
 
         output_string = None
@@ -169,12 +168,9 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou
         else:
             (output_string, err_string) = proc.communicate()
         if output_string:
-            if binary_output:
-                output_lines = [output_string]
-            else:
-                if output_string[-1] != "\n":
-                    output_string = output_string + "\n"
-                output_lines = output_string.splitlines(True)
+            if output_string[-1] != "\n":
+                output_string = output_string + "\n"
+            output_lines = output_string.splitlines(True)
 
             if log_output:
                 with program_log_lock:
@@ -207,7 +203,7 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_ou
     return (proc.returncode, output_string)
 
 def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune=None,
-                     log_output=True, binary_output=False, raise_err=False, callback=None,
+                     log_output=True, raise_err=False, callback=None,
                      env_add=None, reset_handlers=True, reset_lang=True):
     """ Run an external program and redirect the output to a file.
 
@@ -218,7 +214,6 @@ def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune
         :param root: The directory to chroot to before running command.
         :param env_prune: environment variable to remove before execution
         :param log_output: whether to log the output of command
-        :param binary_output: whether to treat the output of command as binary data
         :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero
         :param callback: method to call while waiting for process to finish, passed Popen object
         :param env_add: environment variables to add before execution
@@ -228,7 +223,7 @@ def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune
     """
     argv = [command] + list(argv)
     return _run_program(argv, stdin=stdin, stdout=stdout, root=root, env_prune=env_prune,
-            log_output=log_output, binary_output=binary_output, raise_err=raise_err, callback=callback,
+            log_output=log_output, raise_err=raise_err, callback=callback,
             env_add=env_add, reset_handlers=reset_handlers, reset_lang=reset_lang)[0]
 
 def execWithCapture(command, argv, stdin=None, root='/', log_output=True, filter_stderr=False,
@@ -290,7 +285,7 @@ def __init__(self, proc, argv, callback):
             self._proc = proc
             self._argv = argv
             self._callback = callback
-            self._data = ""
+            self._data = b""
 
         def __iter__(self):
             return self
@@ -311,12 +306,12 @@ def __next__(self):
                 if select.select([self._proc.stdout], [], [], 0)[0]:
                     size = len(self._proc.stdout.peek(1))
                     if size > 0:
-                        self._data += self._proc.stdout.read(size).decode("utf-8")
+                        self._data += self._proc.stdout.read(size)
 
-                if self._data.find("\n") >= 0:
-                    line = self._data.split("\n", 1)
+                if self._data.find(b"\n") >= 0:
+                    line = self._data.split(b"\n", 1)
                     self._data = line[1]
-                    return line[0]
+                    return line[0].decode("utf-8", "ignore")
 
                 if self._proc.poll() is not None or not self._callback(self._proc):
                     # Output finished, wait 60s for the process to end

tests/pylorax/test_executils.py

@@ -57,7 +57,7 @@ def test_execWithRedirect(self):
         program_log.setLevel(logging.INFO)
 
         tmp_f = tempfile.NamedTemporaryFile(prefix="lorax.test.log.", delete=False)
-        fh = logging.FileHandler(filename=tmp_f.name, mode="w")
+        fh = logging.FileHandler(filename=tmp_f.name, mode="w", encoding="UTF-8")
         program_log.addHandler(fh)
 
         try:
@@ -66,7 +66,7 @@ def test_execWithRedirect(self):
             self.assertEqual(rc, 1)
 
             fh.close()
-            with open(tmp_f.name, "r") as f:
+            with open(tmp_f.name, "r", encoding="UTF-8") as f:
                 logged_text = f.readlines()[-1].strip()
             self.assertEqual(logged_text, "The Once-ler was here.")
         finally: