From 8ff89c33b29237889bbfbcb1534fe04d0a199fac Mon Sep 17 00:00:00 2001 From: Bogaert Aaron Date: Wed, 10 Sep 2025 16:26:19 +0200 Subject: [PATCH 1/5] changed pythainer to allow copy --- src/pythainer/builders/__init__.py | 62 +++++++++++++++--------------- src/pythainer/builders/cmds.py | 48 +++++++++++++++++++++++ 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/src/pythainer/builders/__init__.py b/src/pythainer/builders/__init__.py index 04993c2..10a24db 100644 --- a/src/pythainer/builders/__init__.py +++ b/src/pythainer/builders/__init__.py @@ -13,6 +13,7 @@ from pythainer.builders.cmds import ( AddPkgDockerBuildCommand, + CopyDockerBuildCommand, DockerBuildCommand, StrDockerBuildCommand, ) @@ -29,6 +30,7 @@ def _generate_dockerfile_content( package_manager: str, + docker_file_Path: Path, commands: List[DockerBuildCommand], ) -> str: """ @@ -44,7 +46,7 @@ def _generate_dockerfile_content( str: The generated Dockerfile content as a string. """ joined_lines = "\n".join( - c.get_str_for_dockerfile(pkg_manager=package_manager) for c in commands + c.get_str_for_dockerfile(pkg_manager=package_manager,docker_file_Path=docker_file_Path) for c in commands ) file_content = joined_lines.strip() + "\n" return file_content @@ -220,7 +222,7 @@ def add_packages(self, packages: List[str]) -> None: """ self._build_commands.append(AddPkgDockerBuildCommand(packages=packages)) - def copy(self, filename: PathType, destination: PathType) -> None: + def copy(self, file_path: PathType, destination_path: PathType) -> None: """ Copies a file to the docker container @@ -228,8 +230,7 @@ def copy(self, filename: PathType, destination: PathType) -> None: filename (PathType): The file to copy to the container. destination (PathType): The location to place the file within the Docker container. """ - cmd = f"COPY {filename} {destination}" - self._build_commands.append(StrDockerBuildCommand(cmd)) + self._build_commands.append(CopyDockerBuildCommand(file_path,destination_path)) class DockerBuilder(PartialDockerBuilder): @@ -257,6 +258,7 @@ def __init__( """ super().__init__() self._tag = tag + self._container_dir = Path(f"/tmp/pythainer/docker/{self._tag}/") self._package_manager = package_manager self._use_buildkit = use_buildkit @@ -270,6 +272,7 @@ def generate_dockerfile(self, dockerfile_paths: List[PathType]) -> None: """ dockerfile_content = _generate_dockerfile_content( package_manager=self._package_manager, + docker_file_Path = self._container_dir, commands=self._build_commands, ) @@ -389,34 +392,29 @@ def build(self, dockerfile_savepath: PathType = "", docker_context: PathType = " Parameters: dockerfile_savepath (PathType): Optional path to save the Dockerfile used for the build. """ - main_dir = Path("/tmp/pythainer/docker/") - mkdir(main_dir) - with tempfile.TemporaryDirectory( - prefix="/tmp/pythainer/docker/docker-build-", - dir=main_dir, - ) as temp_dir: - temp_path = Path(temp_dir) - dockerfile_path = (temp_path / "Dockerfile").resolve() - dockerfile_paths = [dockerfile_path] + ( - [dockerfile_savepath] if dockerfile_savepath else [] - ) - self.generate_dockerfile(dockerfile_paths=dockerfile_paths) - - command = self.get_build_commands( - dockerfile_path=dockerfile_path, - docker_build_dir=Path(docker_context).resolve() if docker_context else temp_path, - uid=get_uid(), - gid=get_gid(), - ) - - environment = self.get_build_environment() - - shell_out( - command=command, - current_dir=temp_path, - environment=environment, - output_is_log=True, - ) + + mkdir(self._container_dir) + dockerfile_path = (self._container_dir / "Dockerfile").resolve() + dockerfile_paths = [dockerfile_path] + ( + [dockerfile_savepath] if dockerfile_savepath else [] + ) + self.generate_dockerfile(dockerfile_paths=dockerfile_paths) + + command = self.get_build_commands( + dockerfile_path=dockerfile_path, + docker_build_dir=Path(docker_context).resolve() if docker_context else self._container_dir, + uid=get_uid(), + gid=get_gid(), + ) + + environment = self.get_build_environment() + + shell_out( + command=command, + current_dir=self._container_dir, + environment=environment, + output_is_log=True, + ) def get_runner( self, diff --git a/src/pythainer/builders/cmds.py b/src/pythainer/builders/cmds.py index 2aadb37..9d3706d 100644 --- a/src/pythainer/builders/cmds.py +++ b/src/pythainer/builders/cmds.py @@ -9,6 +9,9 @@ and package managers. """ +import os +from pathlib import Path +import shutil from typing import List @@ -68,6 +71,51 @@ def get_str_for_dockerfile( """ return str(self._str) +class CopyDockerBuildCommand(DockerBuildCommand): + """ + Represents a simple string command in a Dockerfile, such as a comment or other directive that + does not involve complex logic or conditional behavior. + """ + + def __init__(self, source_path:Path ,destination_path:Path) -> None: + """ + Initializes the StrDockerBuildCommand with a string. + + Parameters: + s (str): The string that represents this Dockerfile command. + """ + super().__init__() + self._source_path = source_path + self._destination_path = destination_path + + def get_str_for_dockerfile( + self, + docker_file_Path: Path, + *args, + **kwargs, + ) -> str: + """ + Returns the string that was initialized at the creation of the object. + + Returns: + str: The command string. + """ + + # TODO: is mkdir needed and if so where? + + if os.path.isfile(self._source_path): + shutil.copyfile(self._source_path, docker_file_Path / self._source_path) + elif os.path.isdir(self._source_path): + shutil.copytree(self._source_path, docker_file_Path / self._source_path,dirs_exist_ok=True) + else: + raise FileExistsError(f'{self._source_path} is not a valid target to copy into the docker container') + + + + cmd = f"COPY --chown=${{USER_NAME}} {self._source_path} {self._destination_path}" + + return cmd + class AddPkgDockerBuildCommand(DockerBuildCommand): """ From 942da6ba1d421bb31044d864b55099254d3c49a3 Mon Sep 17 00:00:00 2001 From: Bogaert Aaron Date: Wed, 10 Sep 2025 16:32:17 +0200 Subject: [PATCH 2/5] clean up of types and names --- src/pythainer/builders/__init__.py | 8 ++++---- src/pythainer/builders/cmds.py | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/pythainer/builders/__init__.py b/src/pythainer/builders/__init__.py index 10a24db..96aa9ba 100644 --- a/src/pythainer/builders/__init__.py +++ b/src/pythainer/builders/__init__.py @@ -222,15 +222,15 @@ def add_packages(self, packages: List[str]) -> None: """ self._build_commands.append(AddPkgDockerBuildCommand(packages=packages)) - def copy(self, file_path: PathType, destination_path: PathType) -> None: + def copy(self, source_path: Path, destination_path: Path) -> None: """ Copies a file to the docker container Parameters: - filename (PathType): The file to copy to the container. - destination (PathType): The location to place the file within the Docker container. + source_path (Path): The file or folder to copy to the container. + destination_path (Path): The location to place the file or folder within the Docker container. """ - self._build_commands.append(CopyDockerBuildCommand(file_path,destination_path)) + self._build_commands.append(CopyDockerBuildCommand(source_path,destination_path)) class DockerBuilder(PartialDockerBuilder): diff --git a/src/pythainer/builders/cmds.py b/src/pythainer/builders/cmds.py index 9d3706d..523040d 100644 --- a/src/pythainer/builders/cmds.py +++ b/src/pythainer/builders/cmds.py @@ -88,6 +88,7 @@ def __init__(self, source_path:Path ,destination_path:Path) -> None: self._source_path = source_path self._destination_path = destination_path + # pylint: disable=arguments-differ def get_str_for_dockerfile( self, docker_file_Path: Path, @@ -101,8 +102,6 @@ def get_str_for_dockerfile( str: The command string. """ - # TODO: is mkdir needed and if so where? - if os.path.isfile(self._source_path): shutil.copyfile(self._source_path, docker_file_Path / self._source_path) elif os.path.isdir(self._source_path): From 96cbaac5b249761cd0230b9cb9d22b692d7aaf57 Mon Sep 17 00:00:00 2001 From: Bogaert Aaron Date: Wed, 10 Sep 2025 17:33:03 +0200 Subject: [PATCH 3/5] Fix for read only files --- src/pythainer/builders/cmds.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pythainer/builders/cmds.py b/src/pythainer/builders/cmds.py index 523040d..3c10c18 100644 --- a/src/pythainer/builders/cmds.py +++ b/src/pythainer/builders/cmds.py @@ -105,6 +105,7 @@ def get_str_for_dockerfile( if os.path.isfile(self._source_path): shutil.copyfile(self._source_path, docker_file_Path / self._source_path) elif os.path.isdir(self._source_path): + shutil.rmtree(docker_file_Path / self._source_path) shutil.copytree(self._source_path, docker_file_Path / self._source_path,dirs_exist_ok=True) else: raise FileExistsError(f'{self._source_path} is not a valid target to copy into the docker container') From 5c4356424bff7f95e8316ac61e837a891d82bc74 Mon Sep 17 00:00:00 2001 From: Bogaert Aaron Date: Thu, 11 Sep 2025 10:02:03 +0200 Subject: [PATCH 4/5] Add python venv builder Signed-off-by: Bogaert Aaron --- src/pythainer/examples/builders/__init__.py | 57 ++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/src/pythainer/examples/builders/__init__.py b/src/pythainer/examples/builders/__init__.py index 82f96fb..c275cac 100644 --- a/src/pythainer/examples/builders/__init__.py +++ b/src/pythainer/examples/builders/__init__.py @@ -7,7 +7,9 @@ projects like CLSPV. """ -from typing import Iterable, List +import os +from pathlib import Path +from typing import Iterable, List, Optional from pythainer.builders import PartialDockerBuilder, UbuntuDockerBuilder from pythainer.builders.utils import cmake_build_install @@ -419,3 +421,56 @@ def qemu_builder( builder.workdir(path="..") return builder + +def pyvenv_builder( + requirements_file_path:Optional[Path] = None, + dependency_folders:Optional[List[Path]] = None, + single_packages: Optional[List[str]] = None, + lib_dir:Path = Path("/home/${USER_NAME}/workspace/libraries") +) -> PartialDockerBuilder: + + """ + Create a python virtual environment given the specified parameters + + Parameters: + requirements_file_path (Optional[Path] = None): + Path to a file structured like a requirements.txt + dependency_folders (Optional[List[Path]] = None): + A list of paths that point towards folders with their own + setup.py or pyproject.toml file, to be added to the venv + single_packages: + A list of names of packages such as + "numpy" or "matplotlib" to be added to the venv. + lib_dir: + lib_dir (Path): Directory for libraries and tools. + + Returns: + PartialDockerBuilder: A builder that, when composed/applied, creates a venv in the container. + """ + + venv_dir = lib_dir / "./python_venv/" + builder = PartialDockerBuilder() + + builder.desc("install the python cvenv") + builder.user() + builder.run("mkdir {lib_dir}") + builder.workdir(venv_dir) + + builder.run("python3 -m venv .cvenv") + builder.run("./.cvenv/bin/python3 -m pip install --upgrade setuptools pip") + + if single_packages: + packagelist = " ".join(single_packages) + builder.run(f"./.cvenv/bin/python3 -m pip install --upgrade {packagelist}") + + if requirements_file_path: + builder.copy(requirements_file_path, venv_dir / "./requirements_file/requirements.txt") + builder.run(f"./.cvenv/bin/python3 -m pip install -r {venv_dir / "./requirements_file/requirements.txt"}") + + if dependency_folders: + for path in dependency_folders: + builder.copy(path,venv_dir / f"./dependencies/{os.path.basename(os.path.normpath(path))}") + builder.run(f"./.cvenv/bin/python3 -m pip install -e ./dependencies/{os.path.basename(os.path.normpath(path))}/") + + builder.env(name="PATH", value=f"$PATH:{venv_dir / ".cvenv/bin/"}") + return builder From fe6a2467478a9cf50572dbf4c1789465813e33b8 Mon Sep 17 00:00:00 2001 From: Bogaert Aaron Date: Mon, 20 Oct 2025 10:04:20 +0200 Subject: [PATCH 5/5] revamp paths Signed-off-by: Bogaert Aaron --- src/pythainer/builders/__init__.py | 56 ++++++++++++++++++------------ src/pythainer/builders/cmds.py | 7 ++-- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/pythainer/builders/__init__.py b/src/pythainer/builders/__init__.py index 96aa9ba..e0251bd 100644 --- a/src/pythainer/builders/__init__.py +++ b/src/pythainer/builders/__init__.py @@ -7,6 +7,7 @@ tailored specifically for Docker environments. """ import os +import shutil import tempfile from pathlib import Path from typing import Dict, List @@ -30,7 +31,6 @@ def _generate_dockerfile_content( package_manager: str, - docker_file_Path: Path, commands: List[DockerBuildCommand], ) -> str: """ @@ -46,7 +46,7 @@ def _generate_dockerfile_content( str: The generated Dockerfile content as a string. """ joined_lines = "\n".join( - c.get_str_for_dockerfile(pkg_manager=package_manager,docker_file_Path=docker_file_Path) for c in commands + c.get_str_for_dockerfile(pkg_manager=package_manager) for c in commands ) file_content = joined_lines.strip() + "\n" return file_content @@ -258,7 +258,6 @@ def __init__( """ super().__init__() self._tag = tag - self._container_dir = Path(f"/tmp/pythainer/docker/{self._tag}/") self._package_manager = package_manager self._use_buildkit = use_buildkit @@ -272,7 +271,6 @@ def generate_dockerfile(self, dockerfile_paths: List[PathType]) -> None: """ dockerfile_content = _generate_dockerfile_content( package_manager=self._package_manager, - docker_file_Path = self._container_dir, commands=self._build_commands, ) @@ -393,28 +391,40 @@ def build(self, dockerfile_savepath: PathType = "", docker_context: PathType = " dockerfile_savepath (PathType): Optional path to save the Dockerfile used for the build. """ - mkdir(self._container_dir) - dockerfile_path = (self._container_dir / "Dockerfile").resolve() - dockerfile_paths = [dockerfile_path] + ( - [dockerfile_savepath] if dockerfile_savepath else [] - ) - self.generate_dockerfile(dockerfile_paths=dockerfile_paths) + main_dir = Path("/tmp/pythainer/docker/") + mkdir(main_dir) + with tempfile.TemporaryDirectory( + prefix="/tmp/pythainer/docker/docker-build-", + dir=main_dir, + ) as temp_dir: - command = self.get_build_commands( - dockerfile_path=dockerfile_path, - docker_build_dir=Path(docker_context).resolve() if docker_context else self._container_dir, - uid=get_uid(), - gid=get_gid(), - ) + temp_path = Path(temp_dir) + dockerfile_path = (temp_path / "Dockerfile").resolve() + dockerfile_paths = [dockerfile_path] + ( + [dockerfile_savepath] if dockerfile_savepath else [] + ) + self.generate_dockerfile(dockerfile_paths=dockerfile_paths) - environment = self.get_build_environment() + data_path = main_dir / "data" + + shutil.move(data_path, temp_path) + + command = self.get_build_commands( + dockerfile_path=dockerfile_path, + docker_build_dir=Path(docker_context).resolve() if docker_context else temp_path, + uid=get_uid(), + gid=get_gid(), + ) + + environment = self.get_build_environment() + + shell_out( + command=command, + current_dir=temp_path, + environment=environment, + output_is_log=True, + ) - shell_out( - command=command, - current_dir=self._container_dir, - environment=environment, - output_is_log=True, - ) def get_runner( self, diff --git a/src/pythainer/builders/cmds.py b/src/pythainer/builders/cmds.py index 3c10c18..ce2ecd1 100644 --- a/src/pythainer/builders/cmds.py +++ b/src/pythainer/builders/cmds.py @@ -102,11 +102,12 @@ def get_str_for_dockerfile( str: The command string. """ + data_path = Path("/tmp/pythainer/docker/data") + if os.path.isfile(self._source_path): - shutil.copyfile(self._source_path, docker_file_Path / self._source_path) + shutil.copyfile(self._source_path, data_path / self._source_path) elif os.path.isdir(self._source_path): - shutil.rmtree(docker_file_Path / self._source_path) - shutil.copytree(self._source_path, docker_file_Path / self._source_path,dirs_exist_ok=True) + shutil.copytree(self._source_path, data_path / self._source_path,dirs_exist_ok=True) else: raise FileExistsError(f'{self._source_path} is not a valid target to copy into the docker container')