diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63e45cf..179a24c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,12 +72,8 @@ jobs: extras: "dev,test" - container: rockylinux:9 python-install: | - dnf install -y python3 python3-pip git curl + dnf install -y python3 python3-pip git extras: "dev,test" - - container: alpine:latest - python-install: | - apk add --no-cache python3 py3-pip git gcc musl-dev python3-dev curl - extras: "dev" # Skip torch tests on Alpine (no wheels available) container: ${{ matrix.container }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 7b83a29..fdb822c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -78,4 +78,8 @@ jobs: - name: Run PyTorch tests run: | .venv\Scripts\activate - pytest tests/test_integration.py -v -k "torch" + python tests/test_integration.py -v + python tests/test_edge_cases.py -v + python tests/test_normalization_integration.py -v + python tests/test_security.py -v + python tests/test_torch_tensor_integration.py -v diff --git a/.gitignore b/.gitignore index d5a70db..de8a166 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,12 @@ cython_debug/ # macOS .DS_Store + +# Test temporary directories +.test_temps/ + +# Benchmark virtual environment +.benchmark_venv/ + +# UV cache directory (for hardlinking optimization) +.uv_cache/ diff --git a/benchmarks/memory_benchmark.py b/benchmarks/memory_benchmark.py index ecc2e5d..f831f4c 100644 --- a/benchmarks/memory_benchmark.py +++ b/benchmarks/memory_benchmark.py @@ -13,6 +13,7 @@ import sys import time from pathlib import Path +from typing import Optional import psutil @@ -78,6 +79,69 @@ def __init__(self): print(f"Failed to initialize NVML on {self.platform}: {e}") self.nvml_initialized = False + # Try to get baseline GPU memory using nvidia-smi as fallback on Windows + if not self.nvml_initialized and self.platform == "Windows": + baseline = self._get_gpu_memory_nvidia_smi() + if baseline is not None: + self.baseline_gpu_memory_mb = baseline + print(f"Using nvidia-smi fallback. Initial GPU memory: {baseline:.1f} MB") + + def _get_gpu_memory_nvidia_smi(self) -> Optional[float]: + """Get GPU memory usage using nvidia-smi command (Windows fallback).""" + try: + import shutil + import subprocess + + # Find nvidia-smi executable + nvidia_smi = shutil.which("nvidia-smi") + if not nvidia_smi: + return None + + # Query GPU memory usage via nvidia-smi + result = subprocess.run( # noqa: S603 + [nvidia_smi, "--query-gpu=memory.used", "--format=csv,nounits,noheader"], + capture_output=True, + text=True, + check=True, + ) + # Parse the output (in MB) + used_mb = float(result.stdout.strip()) + return used_mb + except Exception: + return None + + def _get_gpu_memory_windows_fallback(self, memory_info: dict[str, float]) -> dict[str, float]: + """Fallback method to get GPU memory on Windows using nvidia-smi.""" + current_used = self._get_gpu_memory_nvidia_smi() + if current_used is not None: + memory_info["gpu_used_mb"] = current_used + memory_info["total_vram_mb"] = current_used + + # Calculate delta from baseline + vram_delta = current_used - self.baseline_gpu_memory_mb + memory_info["host_vram_mb"] = max(0, vram_delta) + + # Try to get total GPU memory + try: + import shutil + import subprocess + + nvidia_smi = shutil.which("nvidia-smi") + if nvidia_smi: + result = subprocess.run( # noqa: S603 + [nvidia_smi, "--query-gpu=memory.total", "--format=csv,nounits,noheader"], + capture_output=True, + text=True, + check=True, + ) + memory_info["gpu_total_mb"] = float(result.stdout.strip()) + except Exception as e: + # Log the error for debugging but continue + if self.platform == "Windows": + print(f"Could not get total GPU memory via nvidia-smi: {e}", file=sys.stderr) + + return memory_info + def get_process_tree_pids(self) -> list[int]: """Get all PIDs in the process tree (including children).""" pids = [self.process.pid] @@ -137,20 +201,17 @@ def get_memory_usage(self) -> dict[str, float]: # This is more reliable than per-process tracking, especially on Windows vram_delta = current_used_mb - self.baseline_gpu_memory_mb memory_info["host_vram_mb"] = max(0, vram_delta) - - # Debug output for Windows + except Exception as e: + print(f"Error getting GPU memory usage via NVML: {e}") if self.platform == "Windows": - print( - f"[DEBUG Windows] Current GPU: {current_used_mb:.1f} MB, " - f"Baseline: {self.baseline_gpu_memory_mb:.1f} MB, " - f"Delta: {vram_delta:.1f} MB", - file=sys.stderr, - ) + # Try alternative method for Windows + memory_info = self._get_gpu_memory_windows_fallback(memory_info) - except Exception as e: - print(f"Error getting GPU memory usage: {e}") + # Fallback: Try Windows nvidia-smi if NVML not initialized + elif not self.nvml_initialized and self.platform == "Windows": + memory_info = self._get_gpu_memory_windows_fallback(memory_info) - # Fallback: try PyTorch CUDA memory for current process if NVML failed + # Fallback: try PyTorch CUDA memory for current process if all else failed elif CUDA_AVAILABLE and torch.cuda.is_available(): try: # This only captures current process, but better than nothing diff --git a/install_torch_cuda.py b/install_torch_cuda.py deleted file mode 100644 index 4241788..0000000 --- a/install_torch_cuda.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python3 -""" -Helper script to detect CUDA version and install appropriate PyTorch version. -""" - -import importlib.util -import platform -import re -import subprocess -import sys -from typing import Optional - - -def run_command(cmd: list[str], capture_output: bool = True) -> tuple[int, str, str]: - """Run a command and return exit code, stdout, stderr.""" - try: - if capture_output: - result = subprocess.run(cmd, capture_output=True, text=True, check=False) # noqa: S603 - return result.returncode, result.stdout, result.stderr - else: - result = subprocess.run(cmd, check=False) # noqa: S603 - return result.returncode, "", "" - except Exception as e: - return 1, "", str(e) - - -def get_cuda_version() -> Optional[str]: - """Detect CUDA version from nvidia-smi.""" - # Try nvidia-smi first - returncode, stdout, stderr = run_command( - ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"] - ) - if returncode != 0: - print("nvidia-smi not found or failed. CUDA not available.") - return None - - # Get CUDA version - returncode, stdout, stderr = run_command(["nvidia-smi"]) - if returncode == 0: - # Look for CUDA Version in nvidia-smi output - match = re.search(r"CUDA Version:\s*(\d+\.\d+)", stdout) - if match: - cuda_version = match.group(1) - print(f"Detected CUDA version: {cuda_version}") - return cuda_version - - # Try nvcc as fallback - returncode, stdout, stderr = run_command(["nvcc", "--version"]) - if returncode == 0: - match = re.search(r"release (\d+\.\d+)", stdout) - if match: - cuda_version = match.group(1) - print(f"Detected CUDA version from nvcc: {cuda_version}") - return cuda_version - - print("Could not detect CUDA version") - return None - - -def get_torch_cuda_version(cuda_version: str) -> str: - """Map CUDA version to PyTorch CUDA version.""" - major, minor = cuda_version.split(".") - major = int(major) - minor = int(minor) - - # PyTorch CUDA version mapping (as of late 2024) - if major >= 12: - if minor >= 1: - return "cu121" - else: - return "cu118" # PyTorch typically supports CUDA 11.8 for compatibility - elif major == 11: - if minor >= 8: - return "cu118" - elif minor >= 7: - return "cu117" - else: - return "cu116" - elif major == 10: - return "cu102" - else: - print(f"CUDA {cuda_version} is quite old. Using CPU version.") - return "cpu" - - -def install_torch(cuda_version: Optional[str] = None): - """Install PyTorch with appropriate CUDA support.""" - system = platform.system() - - if cuda_version and cuda_version != "cpu": - torch_cuda = get_torch_cuda_version(cuda_version) - - if torch_cuda == "cpu": - # CPU only - print("\nInstalling PyTorch (CPU only)...") - cmd = [sys.executable, "-m", "pip", "install", "torch", "torchvision", "torchaudio"] - else: - # CUDA version - print(f"\nInstalling PyTorch with CUDA {cuda_version} support (torch_{torch_cuda})...") - - # Different index URLs for different CUDA versions - if torch_cuda in ["cu118", "cu121"]: - index_url = f"https://download.pytorch.org/whl/{torch_cuda}" - else: - index_url = f"https://download.pytorch.org/whl/{torch_cuda}" - - cmd = [ - sys.executable, - "-m", - "pip", - "install", - "torch", - "torchvision", - "torchaudio", - "--index-url", - index_url, - ] - else: - # CPU only - print("\nInstalling PyTorch (CPU only)...") - if system == "Darwin": # macOS - cmd = [sys.executable, "-m", "pip", "install", "torch", "torchvision", "torchaudio"] - else: - cmd = [ - sys.executable, - "-m", - "pip", - "install", - "torch", - "torchvision", - "torchaudio", - "--index-url", - "https://download.pytorch.org/whl/cpu", - ] - - print(f"Running: {' '.join(cmd)}") - returncode, _, _ = run_command(cmd, capture_output=False) - - if returncode != 0: - print("\nERROR: Failed to install PyTorch") - print("You may need to install it manually.") - print("Visit https://pytorch.org/get-started/locally/ for installation instructions.") - return False - - print("\nPyTorch installed successfully!") - return True - - -def verify_torch_installation(): - """Verify PyTorch installation and CUDA availability.""" - try: - import torch - - print(f"\n✓ PyTorch version: {torch.__version__}") - - if torch.cuda.is_available(): - print("✓ CUDA available: Yes") - print(f"✓ CUDA version used by PyTorch: {torch.version.cuda}") - print(f"✓ Number of GPUs: {torch.cuda.device_count()}") - - for i in range(torch.cuda.device_count()): - print(f" GPU {i}: {torch.cuda.get_device_name(i)}") - - # Get memory info - props = torch.cuda.get_device_properties(i) - total_memory = props.total_memory / (1024**3) # Convert to GB - print(f" Memory: {total_memory:.1f} GB") - else: - print("✗ CUDA available: No (CPU only)") - - return True - except ImportError: - print("\n✗ PyTorch is not installed") - return False - except Exception as e: - print(f"\n✗ Error verifying PyTorch: {e}") - return False - - -def main(): - """Main function.""" - print("=" * 60) - print("PyTorch Installation Helper for pyisolate Benchmarks") - print("=" * 60) - - # Check if torch is already installed - torch_spec = importlib.util.find_spec("torch") - if torch_spec is not None: - print("\nPyTorch is already installed.") - verify_torch_installation() - - # Ask if user wants to reinstall - response = input("\nDo you want to reinstall PyTorch? (y/N): ").strip().lower() - if response != "y": - print("Keeping existing PyTorch installation.") - return 0 - else: - print("\nPyTorch is not installed.") - - # Detect CUDA - cuda_version = get_cuda_version() - - if cuda_version: - print(f"\nCUDA {cuda_version} detected.") - response = input("Do you want to install PyTorch with CUDA support? (Y/n): ").strip().lower() - if response == "n": - cuda_version = None - - # Install PyTorch - if install_torch(cuda_version): - # Verify installation - if verify_torch_installation(): - print("\n✓ PyTorch installation complete and verified!") - return 0 - else: - print("\n✗ PyTorch installation verification failed.") - return 1 - else: - print("\n✗ PyTorch installation failed.") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/pyisolate/_internal/host.py b/pyisolate/_internal/host.py index 87a5731..11ae767 100644 --- a/pyisolate/_internal/host.py +++ b/pyisolate/_internal/host.py @@ -334,12 +334,18 @@ def _install_dependencies(self): uv_common_args = [] + # Set up a local cache directory next to venvs to ensure same filesystem + # This enables hardlinking and saves disk space + cache_dir = self.venv_path.parent / ".uv_cache" + cache_dir.mkdir(exist_ok=True) + uv_common_args.extend(["--cache-dir", str(cache_dir)]) + # Install the same version of torch as the current process if self.config["share_torch"]: import torch torch_version = torch.__version__ - if os.name == "nt" and torch_version.endswith("+cpu"): + if torch_version.endswith("+cpu"): # On Windows, the '+cpu' is not included in the version string torch_version = torch_version[:-4] # Remove the '+cpu' suffix cuda_version = torch.version.cuda # type: ignore diff --git a/pyproject.toml b/pyproject.toml index 495f9db..2c955ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,15 +39,23 @@ dev = [ "pyyaml>=5.4.0", # Only for test manifest creation ] test = [ - "torch>=2.0.0", # For testing share_torch functionality "numpy>=1.26.0,<2.0.0", # For testing share_torch functionality + "psutil>=5.9.0", # For memory monitoring + "pytest-asyncio>=0.21.0", # Required for async test fixtures + "pytest>=7.0", # Required by benchmark scripts that import from tests + "pyyaml>=5.4.0", # For test manifest creation + "tabulate>=0.9.0", # For nice output formatting + "torch>=2.0.0", # For testing share_torch functionality ] bench = [ - "torch>=2.0.0", # For tensor benchmarking "numpy>=1.26.0,<2.0.0", # For array benchmarking + "nvidia-ml-py3>=7.352.0", # For GPU memory monitoring "psutil>=5.9.0", # For memory monitoring + "pytest-asyncio>=0.21.0", # Required for async test fixtures + "pytest>=7.0", # Required by benchmark scripts that import from tests + "pyyaml>=5.4.0", # Required by test files that benchmarks import "tabulate>=0.9.0", # For nice output formatting - "nvidia-ml-py3>=7.352.0", # For GPU memory monitoring + "torch>=2.0.0", # For tensor benchmarking ] docs = [ "sphinx>=5.0", diff --git a/run_benchmarks_linux.sh b/run_benchmarks_linux.sh index 69b14ab..4c3faaa 100755 --- a/run_benchmarks_linux.sh +++ b/run_benchmarks_linux.sh @@ -307,7 +307,11 @@ echo "Step 8: Running memory benchmarks..." } >> "../$OUTPUT_FILE" echo "Running memory_benchmark.py (this may take several minutes)..." -python memory_benchmark.py --counts 1,2,5,10 --test-both-modes 2>&1 | tee -a "../$OUTPUT_FILE" || { +echo "Starting at: $(date)" +echo "NOTE: If nothing has changed after 90 minutes, press Ctrl+C" +echo "The test intentionally pushes VRAM limits and may appear frozen when it hits limits." +echo "" +python memory_benchmark.py --counts 1,2,5,10,25,50,100 2>&1 | tee -a "../$OUTPUT_FILE" || { echo "WARNING: Memory benchmark failed or was interrupted" echo "[$(date)] WARNING: Memory benchmark failed" >> "../$OUTPUT_FILE" echo "Exit code: $?" >> "../$OUTPUT_FILE" diff --git a/run_benchmarks_powershell_launcher.bat b/run_benchmarks_powershell_launcher.bat new file mode 100644 index 0000000..bd34f6a --- /dev/null +++ b/run_benchmarks_powershell_launcher.bat @@ -0,0 +1,47 @@ +@echo off +echo ================================================================ +echo PowerShell Benchmark Launcher +echo ================================================================ +echo. +echo This will run the benchmarks using PowerShell, which may work +echo better with CUDA multiprocessing on Windows. +echo. + +REM Check if PowerShell scripts can run +powershell -Command "Get-ExecutionPolicy -Scope CurrentUser" > temp_policy.txt 2>&1 +set /p CURRENT_POLICY= "%OUTPUT_FILE%" echo ================================================================ >> "%OUTPUT_FILE%" @@ -138,42 +146,111 @@ if %ERRORLEVEL% NEQ 0 ( ) echo Virtual environment activated: OK -REM Step 4: Install pyisolate and dependencies +REM Set environment variables that might help with multiprocessing +set PYTHONUNBUFFERED=1 +set CUDA_LAUNCH_BLOCKING=1 +echo Set PYTHONUNBUFFERED=1 and CUDA_LAUNCH_BLOCKING=1 for better error handling + +REM Step 4: Detect CUDA and install PyTorch appropriately echo. -echo Step 4: Installing pyisolate and base dependencies... -uv pip install -e ".[bench]" 2>"%ERROR_LOG%" -if %ERRORLEVEL% NEQ 0 ( +echo Step 4: Detecting GPU and installing PyTorch... +echo. + +REM Check for CUDA availability +set "CUDA_AVAILABLE=0" +set "CUDA_VERSION=" + +where nvidia-smi >nul 2>&1 +if %ERRORLEVEL% EQU 0 ( + echo NVIDIA GPU detected. Checking CUDA version... + + REM Get CUDA version from nvidia-smi + for /f "tokens=*" %%i in ('nvidia-smi 2^>nul ^| findstr "CUDA Version"') do ( + set "CUDA_LINE=%%i" + ) + + REM Extract CUDA version number + if defined CUDA_LINE ( + for /f "tokens=4" %%a in ("!CUDA_LINE!") do ( + set "CUDA_VERSION=%%a" + set "CUDA_AVAILABLE=1" + ) + ) + + if !CUDA_AVAILABLE! EQU 1 ( + echo Detected CUDA version: !CUDA_VERSION! + echo [%date% %time%] CUDA detected: !CUDA_VERSION! >> "%OUTPUT_FILE%" + + REM Determine PyTorch CUDA version based on detected CUDA + REM Extract major version + for /f "tokens=1 delims=." %%a in ("!CUDA_VERSION!") do set "CUDA_MAJOR=%%a" + + REM Map CUDA version to PyTorch index + if !CUDA_MAJOR! GEQ 12 ( + set "TORCH_INDEX=https://download.pytorch.org/whl/cu121" + echo Installing PyTorch with CUDA 12.1 support... + ) else if !CUDA_MAJOR! EQU 11 ( + set "TORCH_INDEX=https://download.pytorch.org/whl/cu118" + echo Installing PyTorch with CUDA 11.8 support... + ) else ( + set "TORCH_INDEX=https://download.pytorch.org/whl/cpu" + echo CUDA version too old, installing CPU-only PyTorch... + ) + ) else ( + echo Could not determine CUDA version, installing CPU-only PyTorch... + set "TORCH_INDEX=https://download.pytorch.org/whl/cpu" + ) +) else ( + echo No NVIDIA GPU detected. Installing CPU-only PyTorch... + set "TORCH_INDEX=https://download.pytorch.org/whl/cpu" + echo [%date% %time%] No CUDA detected, using CPU PyTorch >> "%OUTPUT_FILE%" +) + +REM Install PyTorch with appropriate index +echo. +echo Installing PyTorch from: !TORCH_INDEX! +uv pip install torch torchvision torchaudio --index-url !TORCH_INDEX! > "%TEMP_OUTPUT%" 2>&1 +set TORCH_INSTALL_RESULT=%ERRORLEVEL% +type "%TEMP_OUTPUT%" +type "%TEMP_OUTPUT%" >> "%OUTPUT_FILE%" + +if %TORCH_INSTALL_RESULT% NEQ 0 ( + echo. + echo ERROR: Failed to install PyTorch. + echo [%date% %time%] ERROR: Failed to install PyTorch >> "%OUTPUT_FILE%" + echo. + echo Continuing without PyTorch - some benchmarks will be skipped + echo. +) else ( + echo PyTorch installed successfully! + echo [%date% %time%] PyTorch installed successfully >> "%OUTPUT_FILE%" +) + +REM Step 5: Install remaining dependencies +echo. +echo Step 5: Installing remaining dependencies... + +REM Install benchmark dependencies +uv pip install numpy psutil tabulate nvidia-ml-py3 pytest pytest-asyncio pyyaml > "%TEMP_OUTPUT%" 2>&1 +type "%TEMP_OUTPUT%" +type "%TEMP_OUTPUT%" >> "%OUTPUT_FILE%" + +REM Install pyisolate in editable mode +uv pip install -e . > "%TEMP_OUTPUT%" 2>&1 +set PYISOLATE_RESULT=%ERRORLEVEL% +type "%TEMP_OUTPUT%" +type "%TEMP_OUTPUT%" >> "%OUTPUT_FILE%" + +if %PYISOLATE_RESULT% NEQ 0 ( echo ERROR: Failed to install pyisolate - type "%ERROR_LOG%" echo [%date% %time%] ERROR: Failed to install pyisolate >> "%OUTPUT_FILE%" - type "%ERROR_LOG%" >> "%OUTPUT_FILE%" pause exit /b 1 ) echo pyisolate installed: OK echo [%date% %time%] pyisolate installed >> "%OUTPUT_FILE%" -REM Step 5: Install PyTorch with correct CUDA version -echo. -echo Step 5: Installing PyTorch with appropriate CUDA support... -echo Running PyTorch installation helper... -python install_torch_cuda.py 2>"%ERROR_LOG%" -if %ERRORLEVEL% NEQ 0 ( - echo WARNING: PyTorch installation helper failed - echo Attempting manual CPU-only installation... - uv pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu 2>"%ERROR_LOG%" - if !ERRORLEVEL! NEQ 0 ( - echo ERROR: Failed to install PyTorch - type "%ERROR_LOG%" - echo [%date% %time%] ERROR: Failed to install PyTorch >> "%OUTPUT_FILE%" - type "%ERROR_LOG%" >> "%OUTPUT_FILE%" - echo. - echo Continuing without PyTorch - some benchmarks will be skipped - ) -) -echo [%date% %time%] PyTorch installation completed >> "%OUTPUT_FILE%" - -REM Verify Python and package versions +REM Step 6: Verify installation echo. echo Step 6: Verifying installation... echo. >> "%OUTPUT_FILE%" @@ -192,7 +269,7 @@ echo Step 7: Running performance benchmarks... echo ================================================================ >> "%OUTPUT_FILE%" echo PERFORMANCE BENCHMARKS >> "%OUTPUT_FILE%" echo ================================================================ >> "%OUTPUT_FILE%" -echo. +echo. >> "%OUTPUT_FILE%" cd benchmarks 2>nul if %ERRORLEVEL% NEQ 0 ( @@ -203,14 +280,38 @@ if %ERRORLEVEL% NEQ 0 ( ) echo Running benchmark.py (this may take several minutes)... -python benchmark.py --quick 2>&1 | tee -a "..\%OUTPUT_FILE%" -if %ERRORLEVEL% NEQ 0 ( +echo Output is being saved to the results file... + +REM Run benchmark directly without start command +echo Starting performance benchmark... +python benchmark.py --quick > "%TEMP_OUTPUT%" 2> "%ERROR_LOG%" +set BENCHMARK_RESULT=!ERRORLEVEL! + +REM Combine stdout and stderr +if exist "%ERROR_LOG%" ( + type "%ERROR_LOG%" >> "%TEMP_OUTPUT%" +) + +REM Display and save the output +if exist "%TEMP_OUTPUT%" ( + type "%TEMP_OUTPUT%" + type "%TEMP_OUTPUT%" >> "..\%OUTPUT_FILE%" +) + +if %BENCHMARK_RESULT% EQU -1 ( + echo. + echo WARNING: Performance benchmark timed out after 10 minutes + echo This may indicate CUDA multiprocessing issues on Windows + echo Partial results have been saved + echo. +) else if %BENCHMARK_RESULT% NEQ 0 ( + echo. echo WARNING: Performance benchmark failed or was interrupted echo [%date% %time%] WARNING: Performance benchmark failed >> "..\%OUTPUT_FILE%" - echo Error code: %ERRORLEVEL% >> "..\%OUTPUT_FILE%" + echo Error code: %BENCHMARK_RESULT% >> "..\%OUTPUT_FILE%" echo. >> "..\%OUTPUT_FILE%" - echo Continuing with memory benchmarks... ) +echo Continuing with memory benchmarks... REM Step 8: Run memory benchmarks echo. @@ -219,14 +320,70 @@ echo. >> "..\%OUTPUT_FILE%" echo ================================================================ >> "..\%OUTPUT_FILE%" echo MEMORY BENCHMARKS >> "..\%OUTPUT_FILE%" echo ================================================================ >> "..\%OUTPUT_FILE%" -echo. +echo. >> "..\%OUTPUT_FILE%" echo Running memory_benchmark.py (this may take several minutes)... -python memory_benchmark.py --counts 1,2,5,10 --test-both-modes 2>&1 | tee -a "..\%OUTPUT_FILE%" -if %ERRORLEVEL% NEQ 0 ( +echo Output is being saved to the results file... +echo NOTE: This test intentionally pushes VRAM limits to find maximum capacity + +REM Run memory benchmark +REM Memory benchmarks take longer due to multiple extension creation + +REM First, let's make sure we're in the benchmarks directory before running +REM and calculate the timeout time right before we need it + +echo Starting memory benchmark preparation... + +REM Run the memory benchmark with real-time output +echo. +echo ================================================================ +echo STARTING MEMORY BENCHMARK +echo ================================================================ +echo Starting at: !time! +echo If nothing has changed after 90 minutes, press Ctrl+C +echo. +echo The test intentionally pushes VRAM limits to find maximum capacity. +echo CUDA out-of-memory errors are EXPECTED and part of the testing. +echo. + +REM Also log to file +echo. >> "..\%OUTPUT_FILE%" +echo ================================================================ >> "..\%OUTPUT_FILE%" +echo STARTING MEMORY BENCHMARK >> "..\%OUTPUT_FILE%" +echo ================================================================ >> "..\%OUTPUT_FILE%" +echo Starting at: !time! >> "..\%OUTPUT_FILE%" +echo. >> "..\%OUTPUT_FILE%" + +REM Now run the actual benchmark and capture output +echo Running: python memory_benchmark.py --counts 1,2,5,10,25,50,100 +echo. + +REM Run the benchmark and capture output to a temporary file +python memory_benchmark.py --counts 1,2,5,10,25,50,100 > memory_output.tmp 2> memory_errors.tmp +set MEMORY_RESULT=!ERRORLEVEL! + +REM Combine stdout and stderr +if exist memory_errors.tmp ( + type memory_errors.tmp >> memory_output.tmp + del memory_errors.tmp +) + +REM Display the output to console +if exist memory_output.tmp ( + type memory_output.tmp + REM Also append to the main output file + type memory_output.tmp >> "..\%OUTPUT_FILE%" + del memory_output.tmp +) + +if %MEMORY_RESULT% NEQ 0 ( + echo. echo WARNING: Memory benchmark failed or was interrupted echo [%date% %time%] WARNING: Memory benchmark failed >> "..\%OUTPUT_FILE%" - echo Error code: %ERRORLEVEL% >> "..\%OUTPUT_FILE%" + echo Error code: %MEMORY_RESULT% >> "..\%OUTPUT_FILE%" + echo. + echo NOTE: Check the output above for details. + echo If the test appeared stuck, it may be due to Windows CUDA multiprocessing issues. ) cd .. @@ -266,6 +423,10 @@ echo ================================================================ >> "%OUTPU echo [%date% %time%] Benchmark collection completed >> "%OUTPUT_FILE%" echo ================================================================ >> "%OUTPUT_FILE%" +REM Cleanup temporary files +if exist "%TEMP_OUTPUT%" del "%TEMP_OUTPUT%" +if exist "%ERROR_LOG%" del "%ERROR_LOG%" + REM Deactivate virtual environment call deactivate 2>nul @@ -279,6 +440,12 @@ echo Results have been saved to: %OUTPUT_FILE% echo. echo Please send the file '%OUTPUT_FILE%' back for analysis. echo. +echo IMPORTANT NOTES: +echo - The benchmarks intentionally push VRAM limits to find maximum capacity +echo - CUDA out-of-memory errors are EXPECTED and part of the testing +echo - If tests timeout, it may indicate Windows CUDA multiprocessing limitations +echo - Partial results are still valuable and have been saved +echo. echo If you encountered any errors, please also include any error echo messages shown above. echo. diff --git a/run_benchmarks_windows.ps1 b/run_benchmarks_windows.ps1 new file mode 100644 index 0000000..41e1edf --- /dev/null +++ b/run_benchmarks_windows.ps1 @@ -0,0 +1,279 @@ +# PyIsolate Benchmark Runner for Windows (PowerShell Version) +# This script runs the same benchmarks as the .bat file but uses PowerShell +# +# HOW TO RUN THIS SCRIPT: +# +# Option 1: Use the launcher (EASIEST) +# Double-click: run_benchmarks_powershell_launcher.bat +# +# Option 2: Enable PowerShell scripts permanently +# 1. Open PowerShell as regular user (not admin) +# 2. Run: Set-ExecutionPolicy -ExecutionPolicy Bypass -Scope CurrentUser +# 3. Type Y and press Enter +# 4. Right-click this .ps1 file and select "Run with PowerShell" +# +# Option 3: Run with temporary bypass (one-time) +# 1. Open PowerShell in this directory +# 2. Run: powershell -ExecutionPolicy Bypass -File .\run_benchmarks_windows.ps1 + +$ErrorActionPreference = "Continue" + +Write-Host "================================================================" -ForegroundColor Cyan +Write-Host "PyIsolate Benchmark Runner for Windows (PowerShell)" -ForegroundColor Cyan +Write-Host "================================================================" -ForegroundColor Cyan +Write-Host "" + +# Set up paths and filenames +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$Timestamp = Get-Date -Format "yyyyMMdd_HHmmss" +$OutputFile = "benchmark_results_${env:COMPUTERNAME}_${Timestamp}.txt" +$VenvDir = ".benchmark_venv" + +# Start output file +"[$(Get-Date)] Starting benchmark process..." | Out-File $OutputFile +"================================================================" | Add-Content $OutputFile +"SYSTEM INFORMATION" | Add-Content $OutputFile +"================================================================" | Add-Content $OutputFile +"System: Windows" | Add-Content $OutputFile +"Computer Name: $env:COMPUTERNAME" | Add-Content $OutputFile +"" | Add-Content $OutputFile + +# Get system information +"Windows Version Details:" | Add-Content $OutputFile +(Get-WmiObject Win32_OperatingSystem).Caption | Add-Content $OutputFile +"" | Add-Content $OutputFile + +# Step 1: Check for uv +Write-Host "Step 1: Checking for uv installation..." +$uvPath = Get-Command uv -ErrorAction SilentlyContinue +if (-not $uvPath) { + Write-Host "" + Write-Host "ERROR: uv is not installed or not in PATH" -ForegroundColor Red + Write-Host "" + Write-Host "Please install uv using one of these methods:" + Write-Host "" + Write-Host "Option 1: Using PowerShell (recommended):" -ForegroundColor Yellow + Write-Host ' irm https://astral.sh/uv/install.ps1 | iex' + Write-Host "" + Write-Host "Option 2: Using pip:" + Write-Host " pip install uv" + Write-Host "" + Write-Host "After installation, please restart this script." + Write-Host "" + "[$(Get-Date)] ERROR: uv not found" | Add-Content $OutputFile + Read-Host "Press Enter to exit" + exit 1 +} +Write-Host "uv found: OK" -ForegroundColor Green +"[$(Get-Date)] uv found" | Add-Content $OutputFile + +# Step 2: Create virtual environment +Write-Host "" +Write-Host "Step 2: Creating virtual environment..." +if (Test-Path $VenvDir) { + Write-Host "Removing existing virtual environment..." + Remove-Item -Recurse -Force $VenvDir -ErrorAction SilentlyContinue +} + +& uv venv $VenvDir 2>&1 | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Failed to create virtual environment" -ForegroundColor Red + "[$(Get-Date)] ERROR: Failed to create venv" | Add-Content $OutputFile + Read-Host "Press Enter to exit" + exit 1 +} +Write-Host "Virtual environment created: OK" -ForegroundColor Green +"[$(Get-Date)] Virtual environment created" | Add-Content $OutputFile + +# Step 3: Activate virtual environment +Write-Host "" +Write-Host "Step 3: Activating virtual environment..." +& "$VenvDir\Scripts\Activate.ps1" +Write-Host "Virtual environment activated: OK" -ForegroundColor Green + +# Step 4: Detect CUDA and install PyTorch +Write-Host "" +Write-Host "Step 4: Detecting GPU and installing PyTorch..." +Write-Host "" + +$cudaAvailable = $false +$torchIndex = "https://download.pytorch.org/whl/cpu" + +# Check for CUDA +$nvidiaSmi = Get-Command nvidia-smi -ErrorAction SilentlyContinue +if ($nvidiaSmi) { + Write-Host "NVIDIA GPU detected. Checking CUDA version..." + $cudaInfo = & nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>$null + if ($LASTEXITCODE -eq 0) { + $cudaVersion = (& nvidia-smi | Select-String "CUDA Version" | ForEach-Object { $_ -match "CUDA Version:\s*(\d+\.\d+)" | Out-Null; $matches[1] }) + if ($cudaVersion) { + Write-Host "Detected CUDA version: $cudaVersion" -ForegroundColor Green + "[$(Get-Date)] CUDA detected: $cudaVersion" | Add-Content $OutputFile + + $cudaMajor = [int]($cudaVersion.Split('.')[0]) + if ($cudaMajor -ge 12) { + $torchIndex = "https://download.pytorch.org/whl/cu121" + Write-Host "Installing PyTorch with CUDA 12.1 support..." + } elseif ($cudaMajor -eq 11) { + $torchIndex = "https://download.pytorch.org/whl/cu118" + Write-Host "Installing PyTorch with CUDA 11.8 support..." + } + } + } +} else { + Write-Host "No NVIDIA GPU detected. Installing CPU-only PyTorch..." + "[$(Get-Date)] No CUDA detected, using CPU PyTorch" | Add-Content $OutputFile +} + +Write-Host "" +Write-Host "Installing PyTorch from: $torchIndex" +# Suppress PowerShell's stderr warnings for this command +$ErrorActionPreference = "SilentlyContinue" +$output = & uv pip install torch torchvision torchaudio --index-url $torchIndex 2>&1 +$ErrorActionPreference = "Continue" +$output | Out-String | Tee-Object -Append $OutputFile +if ($LASTEXITCODE -ne 0) { + Write-Host "" + Write-Host "ERROR: Failed to install PyTorch." -ForegroundColor Red + "[$(Get-Date)] ERROR: Failed to install PyTorch" | Add-Content $OutputFile + Write-Host "Continuing without PyTorch - some benchmarks will be skipped" +} else { + Write-Host "PyTorch installed successfully!" -ForegroundColor Green + "[$(Get-Date)] PyTorch installed successfully" | Add-Content $OutputFile +} + +# Step 5: Install remaining dependencies +Write-Host "" +Write-Host "Step 5: Installing remaining dependencies..." + +$ErrorActionPreference = "SilentlyContinue" +$output = & uv pip install numpy psutil tabulate nvidia-ml-py3 pytest pytest-asyncio pyyaml 2>&1 +$ErrorActionPreference = "Continue" +$output | Out-String | Tee-Object -Append $OutputFile + +$ErrorActionPreference = "SilentlyContinue" +$output = & uv pip install -e . 2>&1 +$ErrorActionPreference = "Continue" +$output | Out-String | Tee-Object -Append $OutputFile +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Failed to install pyisolate" -ForegroundColor Red + "[$(Get-Date)] ERROR: Failed to install pyisolate" | Add-Content $OutputFile + Read-Host "Press Enter to exit" + exit 1 +} +Write-Host "pyisolate installed: OK" -ForegroundColor Green +"[$(Get-Date)] pyisolate installed" | Add-Content $OutputFile + +# Step 6: Verify installation +Write-Host "" +Write-Host "Step 6: Verifying installation..." +"" | Add-Content $OutputFile +"Package Versions:" | Add-Content $OutputFile +& python --version 2>&1 | Add-Content $OutputFile +& python -c "import pyisolate; print(f'pyisolate: {pyisolate.__version__}')" 2>&1 | Add-Content $OutputFile +& python -c "import numpy; print(f'numpy: {numpy.__version__}')" 2>&1 | Add-Content $OutputFile +& python -c "import torch; print(f'torch: {torch.__version__}')" 2>&1 | Add-Content $OutputFile +& python -c "import torch; print(f'CUDA available: {torch.cuda.is_available()}')" 2>&1 | Add-Content $OutputFile +& python -c "import psutil; print(f'psutil: {psutil.__version__}')" 2>&1 | Add-Content $OutputFile +"" | Add-Content $OutputFile + +# Step 7: Run performance benchmarks +Write-Host "" +Write-Host "Step 7: Running performance benchmarks..." +"================================================================" | Add-Content $OutputFile +"PERFORMANCE BENCHMARKS" | Add-Content $OutputFile +"================================================================" | Add-Content $OutputFile +"" | Add-Content $OutputFile + +Set-Location benchmarks +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: benchmarks directory not found" -ForegroundColor Red + Write-Host "Make sure you're running this script from the pyisolate root directory" + Read-Host "Press Enter to exit" + exit 1 +} + +Write-Host "Running benchmark.py (this may take several minutes)..." +Write-Host "Output is being saved to the results file..." + +# Run benchmark - PowerShell handles subprocess differently +$env:PYTHONUNBUFFERED = "1" +$output = & python benchmark.py --quick 2>&1 | Out-String +$benchmarkResult = $LASTEXITCODE +$output | Tee-Object -Append "..\$OutputFile" + +if ($benchmarkResult -ne 0) { + Write-Host "" + Write-Host "WARNING: Performance benchmark failed or was interrupted" -ForegroundColor Yellow + "[$(Get-Date)] WARNING: Performance benchmark failed" | Add-Content "..\$OutputFile" + "Error code: $benchmarkResult" | Add-Content "..\$OutputFile" + "" | Add-Content "..\$OutputFile" + Write-Host "Continuing with memory benchmarks..." +} + +# Step 8: Run memory benchmarks +Write-Host "" +Write-Host "Step 8: Running memory benchmarks..." +"" | Add-Content "..\$OutputFile" +"================================================================" | Add-Content "..\$OutputFile" +"MEMORY BENCHMARKS" | Add-Content "..\$OutputFile" +"================================================================" | Add-Content "..\$OutputFile" +"" | Add-Content "..\$OutputFile" + +Write-Host "Running memory_benchmark.py (this may take several minutes)..." +Write-Host "NOTE: This test intentionally pushes VRAM limits to find maximum capacity" + +Write-Host "Starting memory benchmark at $(Get-Date -Format 'h:mm:ss tt')..." +Write-Host "NOTE: If nothing has changed after 90 minutes, press Ctrl+C" -ForegroundColor Yellow +Write-Host "The test intentionally pushes VRAM limits and may appear frozen when it hits limits." + +# Run memory benchmark +$output = & python memory_benchmark.py --counts 1,2,5,10,25,50,100 2>&1 | Out-String +$memoryResult = $LASTEXITCODE +$output | Tee-Object -Append "..\$OutputFile" + +if ($memoryResult -ne 0) { + Write-Host "" + Write-Host "WARNING: Memory benchmark failed or was interrupted" -ForegroundColor Yellow + "[$(Get-Date)] WARNING: Memory benchmark failed" | Add-Content "..\$OutputFile" + "Error code: $memoryResult" | Add-Content "..\$OutputFile" +} + +Set-Location .. + +# Step 9: Collect additional runtime information +Write-Host "" +Write-Host "Step 9: Collecting additional runtime information..." +"" | Add-Content $OutputFile +"================================================================" | Add-Content $OutputFile +"RUNTIME INFORMATION" | Add-Content $OutputFile +"================================================================" | Add-Content $OutputFile + +# Final summary +"" | Add-Content $OutputFile +"================================================================" | Add-Content $OutputFile +"[$(Get-Date)] Benchmark collection completed" | Add-Content $OutputFile +"================================================================" | Add-Content $OutputFile + +# Deactivate virtual environment +deactivate 2>$null + +# Display completion message +Write-Host "" +Write-Host "================================================================" -ForegroundColor Green +Write-Host "BENCHMARK COLLECTION COMPLETED!" -ForegroundColor Green +Write-Host "================================================================" -ForegroundColor Green +Write-Host "" +Write-Host "Results have been saved to: $OutputFile" -ForegroundColor Yellow +Write-Host "" +Write-Host "Please send the file '$OutputFile' back for analysis." +Write-Host "" +Write-Host "IMPORTANT NOTES:" -ForegroundColor Cyan +Write-Host "- The benchmarks intentionally push VRAM limits to find maximum capacity" +Write-Host "- CUDA out-of-memory errors are EXPECTED and part of the testing" +Write-Host "- If tests timeout, it may indicate Windows CUDA multiprocessing limitations" +Write-Host "- Partial results are still valuable and have been saved" +Write-Host "" +Write-Host "Thank you for running the benchmarks!" +Write-Host "" +Read-Host "Press Enter to exit" diff --git a/tests/test_integration.py b/tests/test_integration.py index aec67b8..98fd43b 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -10,6 +10,7 @@ import os import sys import tempfile +from datetime import datetime from pathlib import Path from typing import Any, Optional, TypedDict, cast @@ -45,10 +46,19 @@ def __init__(self): async def setup_test_environment(self, test_name: str) -> Path: """Set up a temporary test environment.""" - self.temp_dir = tempfile.TemporaryDirectory() - self.test_root = Path(self.temp_dir.name) / test_name + # Create test directories within the project folder instead of system temp + project_root = Path(__file__).parent.parent + test_temps_dir = project_root / ".test_temps" + test_temps_dir.mkdir(exist_ok=True) + + # Add timestamp to avoid conflicts + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") + self.test_root = test_temps_dir / f"{test_name}_{timestamp}" self.test_root.mkdir(parents=True, exist_ok=True) + # Store the path for cleanup + self.temp_dir = None # No longer using TemporaryDirectory + # Create venv root directory venv_root = self.test_root / "extension-venvs" venv_root.mkdir(parents=True, exist_ok=True) @@ -203,9 +213,14 @@ async def cleanup(self): except Exception as e: logging.warning(f"Error stopping extensions: {e}") - # Clean up temp directory - if self.temp_dir: - self.temp_dir.cleanup() + # Clean up test directory manually since we're not using TemporaryDirectory + if self.test_root and self.test_root.exists(): + import shutil + + try: + shutil.rmtree(self.test_root) + except Exception as e: + logging.warning(f"Error removing test directory {self.test_root}: {e}") @pytest.mark.asyncio