Skip to content

Fix Issue #15: Implement chunked communication for large optimization problems #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ohtaman wants to merge 2 commits into from
Closed

Fix Issue #15: Implement chunked communication for large optimization problems #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8610125
Fix Issue #15: Implement chunked communication for large optimization…
ohtaman Aug 3, 2025
96420b3
Simplify Issue #15 fix: Use Pyodide filesystem mounting instead of ch…
ohtaman Aug 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 117 additions & 72 deletions src/mip_mcp/executor/pyodide_executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,11 @@ def __init__(self, config: dict[str, Any]):
"execution_timeout", 60.0
) # Execution timeout for MCP

logger.info("Pyodide executor initialized")
# Create isolated temporary directory for this executor instance
self.temp_dir = tempfile.mkdtemp(prefix="mip_mcp_executor_")
logger.info(
f"Pyodide executor initialized with isolated temp dir: {self.temp_dir}"
)

def set_progress_callback(
self, callback: Callable[[SolverProgress], None] | None
Expand Down Expand Up @@ -213,6 +217,21 @@ async def _initialize_pyodide(self) -> None:
f"Pyodide initialization failed: {init_result.get('error', 'unknown error')}"
)

# Mount the isolated temp directory
mount_result = await self._communicate_with_pyodide(
{"action": "mount", "path": self.temp_dir}
)

if not mount_result.get("success"):
logger.warning(
f"Failed to mount temp directory: {mount_result.get('error')}"
)
# Continue anyway, fallback to virtual filesystem
else:
logger.info(
f"Mounted temp directory {self.temp_dir} to /mnt in Pyodide"
)

self._pyodide_initialized = True
logger.info("Pyodide environment initialized successfully")

Expand Down Expand Up @@ -363,6 +382,17 @@ def _get_pyodide_script(self, pyodide_path: str) -> str:
}
}

async function mountTempDir(tempDirPath) {
try {
// Mount the host temp directory to /mnt in Pyodide filesystem
pyodide.FS.mkdir('/mnt');
pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: tempDirPath }, '/mnt');
return { success: true };
} catch (error) {
return { success: false, error: error.message };
}
}

async function executePython(code) {
try {
const result = await pyodide.runPythonAsync(code);
Expand Down Expand Up @@ -424,6 +454,9 @@ def _get_pyodide_script(self, pyodide_path: str) -> str:
case 'init':
response = await initPyodide();
break;
case 'mount':
response = await mountTempDir(request.path);
break;
case 'execute':
response = await executePython(request.code);
break;
Expand Down Expand Up @@ -597,22 +630,34 @@ async def execute_mip_code(
detected_library,
)

# Extract content (LP preferred, then MPS)
lp_content = json_data.get("lp_content")
mps_content = json_data.get("mps_content")

# Automatic format detection: LP preferred, then MPS
content = None
# Get file paths from Pyodide execution and map to host filesystem
lp_file_path = json_data.get("lp_file_path")
mps_file_path = json_data.get("mps_file_path")

# Map Pyodide paths (/mnt/...) to host filesystem paths
host_lp_path = None
host_mps_path = None
if lp_file_path and lp_file_path.startswith("/mnt/"):
host_lp_path = str(
Path(self.temp_dir) / lp_file_path[5:]
) # Remove /mnt/ prefix
if mps_file_path and mps_file_path.startswith("/mnt/"):
host_mps_path = str(
Path(self.temp_dir) / mps_file_path[5:]
) # Remove /mnt/ prefix

# Determine which file to use (LP preferred, then MPS)
source_file_path = None
file_format = None
if lp_content:
content = lp_content
if host_lp_path and Path(host_lp_path).exists():
source_file_path = host_lp_path
file_format = "lp"
elif mps_content:
content = mps_content
elif host_mps_path and Path(host_mps_path).exists():
source_file_path = host_mps_path
file_format = "mps"

if not content:
# Check if there were problems but failed to generate content
if not source_file_path:
# Check if there were problems but failed to generate files
problems_info = json_data.get("problems_info", [])
if problems_info:
problem_errors = [
Expand All @@ -622,25 +667,25 @@ async def execute_mip_code(
error_details = "; ".join(problem_errors)
return (
json_data.get("stdout", ""),
f"Problem found but failed to generate content: {error_details}",
f"Problem found but failed to generate files: {error_details}",
None,
detected_library,
)

return (
json_data.get("stdout", ""),
"No optimization file content generated",
"No optimization file generated",
None,
detected_library,
)

# Send progress for file generation
# Send progress for file processing
self._send_progress(
"modeling", f"Generating {file_format.upper()} optimization file"
"modeling", f"Processing {file_format.upper()} optimization file"
)

# Write content to temporary file
temp_file = self._create_temp_file(content, file_format)
# Copy file from isolated directory to a new temporary file for return
temp_file = self._copy_optimization_file(source_file_path, file_format)

# Send final modeling progress
self._send_progress(
Expand Down Expand Up @@ -689,6 +734,9 @@ def _prepare_execution_code(
if data:
data_setup = f"__data__ = {json.dumps(data)}\n"

# Pass isolated temp directory to Pyodide
temp_dir_setup = f"__temp_dir__ = '{self.temp_dir}'\n"

wrapper_code = f"""
import io
import sys
Expand All @@ -702,6 +750,8 @@ def _prepare_execution_code(

# Setup data if provided
{data_setup}
# Setup isolated temp directory
{temp_dir_setup}

def __convert_to_json_safe(obj, visited=None):
'''Convert objects to JSON-safe format, handling tuple keys and complex objects.'''
Expand Down Expand Up @@ -748,47 +798,34 @@ def __convert_to_json_safe(obj, visited=None):
visited.discard(obj_id)

def __extract_problem_info(globals_dict):
'''Extract PuLP problem information and generate LP/MPS content.'''
'''Extract PuLP problem information and write files to mounted filesystem.'''
problems_info = []
lp_content = None
mps_content = None
lp_file_path = None
mps_file_path = None

for name, obj in globals_dict.items():
if hasattr(obj, 'writeLP') and hasattr(obj, 'writeMPS'):
try:
# Generate LP content
import tempfile
import os

# Create temporary files for LP and MPS
with tempfile.NamedTemporaryFile(mode='w', suffix='.lp', delete=False) as lp_file:
lp_path = lp_file.name

with tempfile.NamedTemporaryFile(mode='w', suffix='.mps', delete=False) as mps_file:
mps_path = mps_file.name

# Write LP and MPS files
obj.writeLP(lp_path)
obj.writeMPS(mps_path)
import time

# Read content
with open(lp_path, 'r') as f:
lp_content = f.read()
# Use mounted filesystem at /mnt with unique filenames
unique_id = str(int(time.time() * 1000000) % 100000000)
lp_file_path = f"/mnt/problem_{{unique_id}}.lp"
mps_file_path = f"/mnt/problem_{{unique_id}}.mps"

with open(mps_path, 'r') as f:
mps_content = f.read()

# Clean up temporary files
os.unlink(lp_path)
os.unlink(mps_path)
# Write LP and MPS files to mounted filesystem (accessible from host)
obj.writeLP(lp_file_path)
obj.writeMPS(mps_file_path)

problem_info = {{
'name': name,
'sense': str(getattr(obj, 'sense', 'unknown')),
'status': str(getattr(obj, 'status', 'unknown')),
'num_variables': len(getattr(obj, 'variables', [])),
'num_constraints': len(getattr(obj, 'constraints', [])),
'objective': str(getattr(obj, 'objective', 'none'))
'objective': str(getattr(obj, 'objective', 'none')),
'lp_file_path': lp_file_path,
'mps_file_path': mps_file_path
}}
problems_info.append(problem_info)

Expand All @@ -801,7 +838,7 @@ def __extract_problem_info(globals_dict):
'error': f"Failed to extract problem info: {{e}}"
}})

return problems_info, lp_content, mps_content
return problems_info, lp_file_path, mps_file_path

try:
# Execute user code
Expand All @@ -814,27 +851,15 @@ def __extract_problem_info(globals_dict):
# Extract and serialize all relevant information as JSON
__globals_copy = dict(globals())

# Find and extract PuLP problem information
__problems_info, __lp_content, __mps_content = __extract_problem_info(__globals_copy)

# Convert user variables to JSON-safe format
__variables_info = {{}}
for __var_name in list(__globals_copy.keys()):
if (not __var_name.startswith('_') and
__var_name not in ['pulp', 'io', 'sys', 'tempfile', 'json'] and
not callable(__globals_copy[__var_name])):
try:
__variables_info[__var_name] = __convert_to_json_safe(__globals_copy[__var_name])
except Exception as __e:
__variables_info[__var_name] = f"<conversion_error: {{__e}}>"
# Find and extract PuLP problem information (writes files to virtual fs)
__problems_info, __lp_file_path, __mps_file_path = __extract_problem_info(__globals_copy)

# Create comprehensive result data
# Create result data with file paths (avoids large JSON)
__result_data = {{
'stdout': __stdout__,
'lp_content': __lp_content,
'mps_content': __mps_content,
'lp_file_path': __lp_file_path,
'mps_file_path': __mps_file_path,
'problems_info': __problems_info,
'variables_info': __variables_info,
'execution_status': 'success'
}}

Expand All @@ -852,8 +877,8 @@ def __extract_problem_info(globals_dict):
# Create error result data
__result_data = {{
'stdout': __stdout__,
'lp_content': None,
'mps_content': None,
'lp_file_path': None,
'mps_file_path': None,
'problems_info': [],
'variables_info': {{}},
'execution_status': 'error',
Expand Down Expand Up @@ -917,23 +942,29 @@ def _indent_code(self, code: str, spaces: int) -> str:
indent = " " * spaces
return "\n".join(indent + line for line in code.split("\n"))

def _create_temp_file(self, content: str, format_type: str) -> str:
"""Create temporary file with the given content.
def _copy_optimization_file(self, source_path: str, format_type: str) -> str:
"""Copy optimization file from isolated directory to new temporary file.

Args:
content: File content
source_path: Path to source file in isolated directory
format_type: File format ("lp" or "mps")

Returns:
Path to temporary file
Path to new temporary file
"""
import shutil

suffix = f".{format_type.lower()}"

with tempfile.NamedTemporaryFile(mode="w", suffix=suffix, delete=False) as f:
f.write(content)
temp_path = f.name

logger.info(f"Created temporary {format_type.upper()} file: {temp_path}")
# Copy the file content
shutil.copy2(source_path, temp_path)

logger.info(
f"Copied {format_type.upper()} file from {source_path} to {temp_path}"
)
return temp_path

async def validate_code(self, code: str) -> dict[str, Any]:
Expand Down Expand Up @@ -1060,6 +1091,20 @@ async def cleanup(self):
self._pyodide_initialized = False
logger.info("Pyodide process cleanup completed")

# Clean up isolated temporary directory
if hasattr(self, "temp_dir") and self.temp_dir:
try:
import shutil

shutil.rmtree(self.temp_dir, ignore_errors=True)
logger.info(f"Cleaned up isolated temp directory: {self.temp_dir}")
except Exception as e:
logger.warning(
f"Failed to clean up temp directory {self.temp_dir}: {e}"
)
finally:
self.temp_dir = None

def __del__(self):
"""Cleanup on destruction."""
if self.pyodide_process and not self._cleanup_started:
Expand Down
21 changes: 10 additions & 11 deletions tests/unit/test_tuple_key_fix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,26 +82,25 @@ async def test_tuple_key_execution_success(self, executor):
executor, "_execute_with_periodic_progress", new_callable=AsyncMock
) as mock_execute,
):
# Mock successful JSON-based execution
# Mock successful filesystem-based execution
mock_execute.return_value = {
"success": True,
"json_data": {
"execution_status": "success",
"stdout": "Tuple key test completed successfully",
"lp_content": "\\* test *\\\nMinimize\nOBJ: x_0_a + x_0_b + x_1_a + x_1_b\nSubject To\n_C1: x_0_a + x_0_b + x_1_a + x_1_b >= 1\nBinaries\nx_0_a\nx_0_b\nx_1_a\nx_1_b\nEnd",
"mps_content": None,
"lp_file_path": "/mnt/problem_12345.lp",
"mps_file_path": "/mnt/problem_12345.mps",
"problems_info": [{"name": "prob", "num_variables": 4}],
"variables_info": {
"var_map": {
"0_a": "x_0_a",
"0_b": "x_0_b",
"1_a": "x_1_a",
"1_b": "x_1_b",
}
},
},
}

# Create mock files in the executor's temp directory
from pathlib import Path

lp_content = "\\* test *\\\nMinimize\nOBJ: x_0_a + x_0_b + x_1_a + x_1_b\nSubject To\n_C1: x_0_a + x_0_b + x_1_a + x_1_b >= 1\nBinaries\nx_0_a\nx_0_b\nx_1_a\nx_1_b\nEnd"
mock_lp_file = Path(executor.temp_dir) / "problem_12345.lp"
mock_lp_file.write_text(lp_content)

stdout, stderr, file_path, library = await executor.execute_mip_code(
tuple_key_code
)
Expand Down