diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 4af9cdd998a..e6c2dc701fa 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -133,12 +133,15 @@ jobs: - name: Run pytest for loading C libraries into the process shell: micromamba-shell {0} run: | + # The path is asked for before the move because the grass command + # does not work from the moved installation. + python_path="$(grass --config python_path)" # The installation is moved away from the prefix it was configured # with, so that the libraries cannot find each other through their # RUNPATH and the loading done by init is the only mechanism left. mv "${HOME}/install" "${HOME}/install-relocated" trap 'mv "${HOME}/install-relocated" "${HOME}/install"' EXIT - PYTHONPATH="${HOME}/install-relocated/etc/python" + PYTHONPATH="${HOME}/install-relocated${python_path#"${HOME}/install"}" export PYTHONPATH pytest python/grass/script/tests/grass_script_setup_load_libs_test.py \ --junitxml=pytest.load_libs.junit.xml diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 34529baac82..201f5a8222d 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -140,12 +140,15 @@ jobs: --junitxml=pytest.gunittest.junit.xml - name: Run pytest for loading C libraries into the process run: | + # The path is asked for before the move because the grass command + # does not work from the moved installation. + python_path="$(grass --config python_path)" # The installation is moved away from the prefix it was configured # with, so that the libraries cannot find each other through their # RUNPATH and the loading done by init is the only mechanism left. mv "${HOME}/install" "${HOME}/install-relocated" trap 'mv "${HOME}/install-relocated" "${HOME}/install"' EXIT - PYTHONPATH="${HOME}/install-relocated/etc/python" + PYTHONPATH="${HOME}/install-relocated${python_path#"${HOME}/install"}" export PYTHONPATH pytest python/grass/script/tests/grass_script_setup_load_libs_test.py \ --junitxml=pytest.load_libs.junit.xml diff --git a/python/grass/app/runtime.py b/python/grass/app/runtime.py index 28c4f5d6a96..6c02d1ebaa0 100644 --- a/python/grass/app/runtime.py +++ b/python/grass/app/runtime.py @@ -356,6 +356,36 @@ def register_library_search_path(install_path): _registered_library_dirs.add(lib_path) +# Variables the C libraries read with getenv: GISBASE to find the files of the +# installation and GISRC to find the session. +LIBRARY_ENVIRONMENT_VARIABLES = ("GISBASE", "GISRC") + + +def set_library_environment(env): + """Set variables from _env_ in the environment which the C libraries read + + The libraries use getenv, which reads the environment of the process, not + the session environment (the _env_ parameter of + :func:`grass.script.setup.init`). On Windows, the C runtime keeps its own + copy of the environment made when the process started, so there even + changes to the global environment do not reach the libraries. + + Without GISBASE, reporting an error fails in the same way as the call + which caused it, and the library ends the process without a message, so + the variables are needed even to see what is wrong. + """ + try: + from grass.lib.gis import G_putenv + except ImportError: + # The grass.lib package is generated during the build and an + # installation may be missing it. + return + for name in LIBRARY_ENVIRONMENT_VARIABLES: + value = env.get(name) + if value: + G_putenv(name, value) + + # Every other GRASS library needs libgrass_gis and libgrass_gis needs # libgrass_datetime, so these two are loaded before the rest. A library loaded # before the GRASS libraries it needs makes the dynamic linker resolve those diff --git a/python/grass/script/setup.py b/python/grass/script/setup.py index 5be02c1448e..df29926f110 100644 --- a/python/grass/script/setup.py +++ b/python/grass/script/setup.py @@ -222,6 +222,7 @@ def setup_runtime_env(gisbase=None, *, env=None, load_libs=False): register_library_search_path, set_dynamic_library_path, set_executable_paths, + set_library_environment, set_path_to_python_executable, set_python_path_variable, RuntimePaths, @@ -263,6 +264,9 @@ def setup_runtime_env(gisbase=None, *, env=None, load_libs=False): # the libraries into the current process for ctypes-based interfaces. register_library_search_path(install_path=gisbase) preload_dynamic_libraries(install_path=gisbase) + # The libraries read the installation path from their own environment. + # A session adds its variables to this later (see init). + set_library_environment(env) set_python_path_variable(install_path=gisbase, env=env) set_path_to_python_executable(env=env) @@ -325,6 +329,15 @@ def init( GRASS C stack and its dependencies (GDAL, PROJ, ...) into the process, which sessions using only tools don't need. + With *load_libs*, the session variables are also set in the environment + of the loaded libraries, which the libraries read for themselves and + which is shared by the whole process. Consequently, the last session + initialized in a process is the one :mod:`grass.lib` works with, even + when the sessions keep their variables in their own environments. + Subprocesses started without an explicit environment inherit that + environment, so they see the session even when it is otherwise limited + to *env*. + When the path or specified mapset does not exist, ValueError is raised. The :func:`get_install_path` function is used to determine where @@ -439,6 +452,12 @@ def init( env["GISRC"] = write_gisrc( mapset_path.directory, mapset_path.location, mapset_path.mapset ) + if load_libs: + # The libraries were loaded before the session file existed, so they + # learn about the session only now. + from grass.app.runtime import set_library_environment + + set_library_environment(env) return SessionHandle(env=env, locked=lock) diff --git a/python/grass/script/tests/grass_script_setup_load_libs_test.py b/python/grass/script/tests/grass_script_setup_load_libs_test.py index efeacee37a4..64296e711cc 100644 --- a/python/grass/script/tests/grass_script_setup_load_libs_test.py +++ b/python/grass/script/tests/grass_script_setup_load_libs_test.py @@ -25,6 +25,8 @@ def run_in_clean_environment(code, tmp_path): The variable is what a parent GRASS session or a manual setup uses to make the GRASS libraries available, so removing it leaves the loading done by init as the only mechanism which can make grass.lib work. + + :returns: the completed process, so that the output streams are available """ source_file = tmp_path / "code.py" source_file.write_text(dedent(code)) @@ -38,8 +40,11 @@ def run_in_clean_environment(code, tmp_path): check=False, env=env, ) - assert result.returncode == 0, result.stderr - return json.loads(result.stdout) + assert result.returncode == 0, ( + f"code failed with {result.returncode}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result @pytest.mark.usefixtures("mock_no_session") @@ -62,16 +67,18 @@ def test_grass_lib_usable_with_load_libs(tmp_path): libraster.Rast_close(fd) print(json.dumps({{"raster_opened": True}})) """ - assert run_in_clean_environment(code, tmp_path=tmp_path)["raster_opened"] + result = run_in_clean_environment(code, tmp_path=tmp_path) + assert json.loads(result.stdout)["raster_opened"] @pytest.mark.usefixtures("mock_no_session") def test_grass_lib_usable_with_load_libs_and_custom_env(tmp_path): """Check that grass.lib is usable with load_libs and a custom environment - The grass.lib loader reads GISBASE from the global environment, so a - session which keeps its variables in its own environment needs the path - to the libraries passed to the loader directly. + A session which keeps its variables in its own environment reaches the + libraries only through the loader and through the environment of the + libraries themselves, because they read neither the session environment + nor, on Windows, the global one. """ project = tmp_path / "test" code = f""" @@ -85,21 +92,56 @@ def test_grass_lib_usable_with_load_libs_and_custom_env(tmp_path): import grass.lib.gis as libgis libgis.G_gisinit(b"test") + session = [ + libgis.G_gisdbase().decode(), + libgis.G_location().decode(), + libgis.G_mapset().decode(), + ] print( json.dumps( {{ - "gis_init_worked": True, + "session_seen_by_c_library": session, "gisbase_in_global_env": "GISBASE" in os.environ, }} ) ) """ - result = run_in_clean_environment(code, tmp_path=tmp_path) - assert result["gis_init_worked"] + result = json.loads(run_in_clean_environment(code, tmp_path=tmp_path).stdout) + # The library works with the session which init created, not with another + # one it may have found in the environment. + assert result["session_seen_by_c_library"] == [ + str(tmp_path), + project.name, + "PERMANENT", + ] # The session did not fall back to the global environment for the lookup. assert not result["gisbase_in_global_env"] +@pytest.mark.usefixtures("mock_no_session") +def test_library_messages_are_reported(tmp_path): + """Check that a message from the C libraries reaches the standard error + + Printing a message needs GISBASE, so a library which does not have it + fails to report the failure as well and ends the process without saying + anything at all. + """ + project = tmp_path / "test" + code = f""" + import os + import grass.script as gs + + gs.create_project(r"{project}") + with gs.setup.init(r"{project}", env=os.environ.copy(), load_libs=True): + import grass.lib.gis as libgis + + libgis.G_warning(b"message from the library") + print("{{}}") + """ + result = run_in_clean_environment(code, tmp_path=tmp_path) + assert "message from the library" in result.stderr + + @pytest.mark.usefixtures("mock_no_session") def test_libraries_not_loaded_by_default(tmp_path, monkeypatch): """Check that a session does not load the C libraries unless asked to"""