Skip to content

Commit 8fac88e

Browse files
JamePengJamePeng
committed
fix(loader): prioritize lib directory for dynamic backend loading
Fix shared library loading order to ensure package bundled libraries are resolved from lib/ before falling back to bin/. - Refactor Windows DLL search path registration - Prevent duplicate DLL directory registration - Preserve lib/ > bin/ priority on Windows - Load bundled libraries before system library fallback - Add backend dependency search path handling - Print resolved library paths for debugging This prevents incorrect DLL resolution for backends with multiple runtime dependencies, such as SYCL. Signed-off-by: JamePeng <jame_peng@sina.com>
1 parent c95c19a commit 8fac88e

1 file changed

Lines changed: 159 additions & 61 deletions

File tree

‎llama_cpp/_ctypes_extensions.py‎

Lines changed: 159 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@
1818
)
1919
from typing_extensions import TypeAlias
2020

21+
_DLL_DIRECTORY_HANDLES = []
22+
_REGISTERED_DLL_DIRECTORIES = set()
23+
2124
def _format_library_dir_contents(base_paths: list[pathlib.Path]) -> str:
2225
"""Format directory contents for diagnostics after library loading fails."""
2326
sections = []
@@ -50,102 +53,197 @@ def _format_library_dir_contents(base_paths: list[pathlib.Path]) -> str:
5053

5154
return "\n".join(sections)
5255

53-
# Load the library
56+
def _add_library_search_paths(paths: list[pathlib.Path]):
57+
"""
58+
Register dynamic library search paths.
59+
60+
Priority:
61+
lib/
62+
bin/
63+
64+
The order is intentionally reversed when registering on Windows
65+
because Windows searches the latest added DLL directory first.
66+
"""
67+
68+
if sys.platform != "win32":
69+
return
70+
71+
# Remove duplicate paths to avoid redundant registration.
72+
unique_paths = []
73+
for path in paths:
74+
p = pathlib.Path(path)
75+
if (
76+
p.exists()
77+
and p.is_dir()
78+
and p not in unique_paths
79+
):
80+
unique_paths.append(p)
81+
82+
# Windows DLL search priority: lib/ > bin/
83+
# The reversed iteration ensures that bin/ directories are registered first,
84+
# since Windows searches the most recently added directory first.
85+
for p in reversed(unique_paths):
86+
p_str = str(p)
87+
if p_str in _REGISTERED_DLL_DIRECTORIES:
88+
continue
89+
handle = os.add_dll_directory(p_str)
90+
_DLL_DIRECTORY_HANDLES.append(handle)
91+
_REGISTERED_DLL_DIRECTORIES.add(p_str)
92+
93+
def _add_optional_backend_paths():
94+
"""
95+
Collect optional backend library paths from environment variables.
96+
97+
Environment variable sources (in priority order):
98+
- CUDA_PATH
99+
- HIP_PATH
100+
- VULKAN_SDK
101+
102+
Each source may contain subdirectories that are added to the search path.
103+
"""
104+
105+
if sys.platform != "win32":
106+
return
107+
108+
backend_paths = []
109+
110+
# Collect CUDA backend paths from environment variable
111+
if "CUDA_PATH" in os.environ:
112+
cuda_path = os.environ["CUDA_PATH"]
113+
for sub_dir in [
114+
"bin", # Default CUDA binaries
115+
os.path.join("bin", "x64"), # 64-bit x86 CUDA binaries
116+
"lib", # Default CUDA libraries
117+
os.path.join("lib", "x64"), # 64-bit x86 CUDA libraries
118+
]:
119+
path = pathlib.Path(cuda_path) / sub_dir
120+
if path.exists():
121+
backend_paths.append(path)
122+
123+
# Collect HIP backend paths from environment variable
124+
if "HIP_PATH" in os.environ:
125+
hip_path = pathlib.Path(os.environ["HIP_PATH"])
126+
backend_paths.extend([
127+
hip_path / "bin", # HIP binaries
128+
hip_path / "lib", # HIP libraries
129+
])
130+
131+
# Collect Vulkan SDK paths from environment variable
132+
if "VULKAN_SDK" in os.environ:
133+
sdk = pathlib.Path(os.environ["VULKAN_SDK"])
134+
backend_paths.extend([
135+
sdk / "Bin", # Vulkan SDK binaries (Windows)
136+
sdk / "Lib", # Vulkan SDK libraries (Windows)
137+
])
138+
139+
_add_library_search_paths(backend_paths)
140+
141+
def _normalize_unique_paths(
142+
paths: list[pathlib.Path]
143+
) -> list[pathlib.Path]:
144+
145+
unique_paths = []
146+
seen = set()
147+
148+
for path in paths:
149+
p = pathlib.Path(path)
150+
151+
if not p.exists() or not p.is_dir():
152+
continue
153+
154+
key = str(p.resolve()).lower()
155+
156+
if key not in seen:
157+
seen.add(key)
158+
unique_paths.append(p)
159+
160+
return unique_paths
161+
162+
# Load a shared library based on platform-specific naming conventions
54163
def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list[pathlib.Path]]):
164+
"""
165+
Load a shared library dynamically by searching through provided paths
166+
and system fallback directories according to the target platform.
167+
168+
Platform-specific behavior:
169+
- Linux/FreeBSD: lib{base}.so
170+
- macOS (Darwin): lib{base}.dylib OR lib{base}.so (fallback)
171+
- Windows: {base}.dll OR lib{base}.dll
172+
173+
Search order:
174+
1. Package bundled libraries from provided paths
175+
2. System-level fallback via find_library()
176+
"""
177+
55178
if isinstance(base_paths, pathlib.Path):
56179
base_paths = [base_paths]
57180

181+
base_paths = _normalize_unique_paths(base_paths)
182+
58183
lib_names = []
59184

185+
# Linux/FreeBSD: use .so extension with system library directories
60186
if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"):
61187
lib_names = [f"lib{lib_base_name}.so"]
62-
63188
base_paths.extend([
64-
"/usr/local/lib",
65-
"/usr/lib",
66-
"/usr/lib64",
189+
"/usr/local/lib", # Local user libraries
190+
"/usr/lib", # System libraries
191+
"/usr/lib64", # 64-bit system libraries (some distros)
67192
])
68193

69194
elif sys.platform == "darwin":
70195
lib_names = [
71-
f"lib{lib_base_name}.dylib",
72-
f"lib{lib_base_name}.so",
196+
f"lib{lib_base_name}.dylib", # macOS native dynamic library
197+
f"lib{lib_base_name}.so", # Fallback for cross-platform compatibility
73198
]
74-
75199
base_paths.extend([
76-
"/usr/local/lib",
77-
"/opt/homebrew/lib",
78-
"/usr/lib",
200+
"/usr/local/lib", # Local user libraries
201+
"/opt/homebrew/lib", # Homebrew installation (macOS)
202+
"/usr/lib", # macOS system libraries
79203
])
80204

81205
elif sys.platform == "win32":
82206
lib_names = [
83-
f"{lib_base_name}.dll",
84-
f"lib{lib_base_name}.dll",
207+
f"{lib_base_name}.dll", # Standard Windows dynamic library
208+
f"lib{lib_base_name}.dll", # Some frameworks use 'lib' prefix
85209
]
86210
else:
87-
raise RuntimeError("Unsupported platform")
211+
raise RuntimeError(f"Unsupported platform: {sys.platform}")
88212

89-
cdll_args = dict() # type: ignore
213+
cdll_args = {}
90214

91-
# Add the library directory to the DLL search path on Windows (if needed)
92215
if sys.platform == "win32":
93-
for base_path in base_paths:
94-
p = pathlib.Path(base_path)
95-
if p.exists() and p.is_dir():
96-
os.add_dll_directory(str(p))
97-
os.environ["PATH"] = str(p) + os.pathsep + os.environ["PATH"]
98-
99-
if sys.platform == "win32" and sys.version_info >= (3, 9):
100-
for base_path in base_paths:
101-
p = pathlib.Path(base_path)
102-
if p.exists() and p.is_dir():
103-
os.add_dll_directory(str(p))
104-
if "CUDA_PATH" in os.environ:
105-
cuda_path = os.environ["CUDA_PATH"]
106-
sub_dirs_to_add = [
107-
"bin",
108-
os.path.join("bin", "x64"), # CUDA 13.0+
109-
"lib",
110-
os.path.join("lib", "x64")
111-
]
112-
for sub_dir in sub_dirs_to_add:
113-
full_path = os.path.join(cuda_path, sub_dir)
114-
if os.path.exists(full_path):
115-
os.add_dll_directory(full_path)
116-
117-
if "HIP_PATH" in os.environ:
118-
os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin"))
119-
os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib"))
120-
121-
if "VULKAN_SDK" in os.environ:
122-
os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Bin"))
123-
os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Lib"))
124-
125-
cdll_args["winmode"] = ctypes.RTLD_GLOBAL
216+
_add_library_search_paths(base_paths)
217+
_add_optional_backend_paths()
218+
if sys.version_info >= (3, 9):
219+
cdll_args["winmode"] = ctypes.RTLD_GLOBAL
126220

127221
errors = []
128222

129-
# First, try to find an available library through the system
130-
lib_path = find_library(lib_base_name)
131-
if lib_path:
132-
try:
133-
return ctypes.CDLL(lib_path, **cdll_args)
134-
except Exception as e:
135-
errors.append(f"{lib_path}: {e}")
136-
137-
# Then fallback to manually checking the list of paths.
223+
# Step 1: Try to load bundled libraries from provided paths
138224
for base_path in base_paths:
139225
for lib_name in lib_names:
140226
lib_path = pathlib.Path(base_path) / lib_name
141-
142227
if lib_path.exists():
143228
try:
144-
return ctypes.CDLL(str(lib_path), **cdll_args)
229+
lib = ctypes.CDLL(str(lib_path), **cdll_args)
230+
print(f"[llama_cpp_python] Loaded {lib_base_name}: {lib_path}")
231+
return lib
145232
except Exception as e:
146233
errors.append(f"{lib_path}: {e}")
147234

148-
# Include directory contents only in the failure path to avoid extra work during successful imports.
235+
# Step 2: System-level fallback using find_library()
236+
lib_path = find_library(lib_base_name)
237+
if lib_path:
238+
try:
239+
lib = ctypes.CDLL(lib_path, **cdll_args)
240+
print(f"[llama_cpp_python] Loaded {lib_base_name}: {lib_path}")
241+
return lib
242+
except Exception as e:
243+
errors.append(f"{lib_path}: {e}")
244+
245+
# Failure path: Only include directory contents to avoid redundant work
246+
# during successful imports.
149247
raise RuntimeError(
150248
f"Failed to load '{lib_base_name}' from {base_paths}\n"
151249
+ "\n".join(errors)

0 commit comments

Comments
 (0)