Merge pull request #71 from mxstack/fix/70-offline-http-cache

Fix #70: Implement HTTP caching for offline mode

Author: jensens

.gitignore

@@ -1,5 +1,6 @@
 *.egg-info
 .*-cache
+.mxdev_cache/
 .coverage
 .coverage.*
 !.coveragerc

CHANGES.md

@@ -2,7 +2,10 @@
 
 ## 5.0.2 (unreleased)
 
-- Nothing yet.
+- Fix #70: HTTP-referenced requirements/constraints files are now properly cached and respected in offline mode. Previously, offline mode only skipped VCS operations but still fetched HTTP URLs. Now mxdev caches all HTTP content in `.mxdev_cache/` during online mode and reuses it during offline mode, enabling true offline operation. This fixes the inconsistent behavior where `-o/--offline` didn't prevent all network activity.
+  [jensens]
+- Improvement: Enhanced help text for `-n/--no-fetch`, `-f/--fetch-only`, and `-o/--offline` command-line options to better explain their differences and when to use each one.
+  [jensens]
 
 ## 5.0.1 (2025-10-23)
 

README.md

@@ -85,7 +85,7 @@ The **main section** must be called `[settings]`, even if kept empty.
 | `default-target` | Target directory for VCS checkouts | `./sources` |
 | `threads` | Number of parallel threads for fetching sources | `4` |
 | `smart-threading` | Process HTTPS packages serially to avoid overlapping credential prompts (see below) | `True` |
-| `offline` | Skip all VCS fetch operations (handy for offline work) | `False` |
+| `offline` | Skip all VCS and HTTP fetches; use cached HTTP content from `.mxdev_cache/` (see below) | `False` |
 | `default-install-mode` | Default `install-mode` for packages: `editable`, `fixed`, or `skip` (see below) | `editable` |
 | `default-update` | Default update behavior: `yes` or `no` | `yes` |
 | `default-use` | Default use behavior (when false, sources not checked out) | `True` |
@@ -103,6 +103,34 @@ This solves the problem where parallel git operations would cause multiple crede
 
 **When to disable**: Set `smart-threading = false` if you have git credential helpers configured (e.g., credential cache, credential store) and never see prompts.
 
+##### Offline Mode and HTTP Caching
+
+When `offline` mode is enabled (or via `-o/--offline` flag), mxdev operates without any network access:
+
+1. **HTTP Caching**: HTTP-referenced requirements/constraints files are automatically cached in `.mxdev_cache/` during online mode
+2. **Offline Usage**: In offline mode, mxdev reads from the cache instead of fetching from the network
+3. **Cache Miss**: If a referenced HTTP file is not in the cache, mxdev will error and prompt you to run in online mode first
+
+**Example workflow:**
+```bash
+# First run in online mode to populate cache
+mxdev
+
+# Subsequent runs can be offline (e.g., on airplane, restricted network)
+mxdev -o
+
+# Cache persists across runs, enabling true offline development
+```
+
+**Cache location**: `.mxdev_cache/` (automatically added to `.gitignore`)
+
+**When to use offline mode**:
+- Working without internet access (airplanes, restricted networks)
+- Testing configuration changes without re-fetching
+- Faster iterations when VCS sources are already checked out
+
+**Note**: Offline mode tolerates missing source directories (logs warnings), while non-offline mode treats missing sources as fatal errors.
+
 #### Package Overrides
 
 ##### `version-overrides`

src/mxdev/main.py

@@ -27,9 +27,19 @@
     type=str,
     default="mx.ini",
 )
-parser.add_argument("-n", "--no-fetch", help="Do not fetch sources", action="store_true")
-parser.add_argument("-f", "--fetch-only", help="Only fetch sources", action="store_true")
-parser.add_argument("-o", "--offline", help="Do not fetch sources, work offline", action="store_true")
+parser.add_argument(
+    "-n",
+    "--no-fetch",
+    help="Skip VCS checkout/update; regenerate files from existing sources (error if missing)",
+    action="store_true",
+)
+parser.add_argument("-f", "--fetch-only", help="Only perform VCS operations; skip file generation", action="store_true")
+parser.add_argument(
+    "-o",
+    "--offline",
+    help="Work offline; skip VCS and HTTP fetches; use cached files (tolerate missing)",
+    action="store_true",
+)
 parser.add_argument(
     "-t",
     "--threads",

src/mxdev/processing.py

@@ -7,16 +7,83 @@
 from urllib import request
 from urllib.error import URLError
 
+import hashlib
 import os
 import typing
 
 
+def _get_cache_key(url: str) -> str:
+    """Generate a deterministic cache key from a URL.
+
+    Uses SHA256 hash of the URL, truncated to 16 hex characters for readability
+    while maintaining low collision probability.
+
+    Args:
+        url: The URL to generate a cache key for
+
+    Returns:
+        16-character hex string (cache key)
+
+    """
+    hash_obj = hashlib.sha256(url.encode("utf-8"))
+    return hash_obj.hexdigest()[:16]
+
+
+def _cache_http_content(url: str, content: str, cache_dir: Path) -> None:
+    """Cache HTTP content to disk.
+
+    Args:
+        url: The URL being cached
+        content: The content to cache
+        cache_dir: Directory to store cache files
+
+    """
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    cache_key = _get_cache_key(url)
+
+    # Write content
+    cache_file = cache_dir / cache_key
+    cache_file.write_text(content, encoding="utf-8")
+
+    # Write URL metadata for debugging
+    url_file = cache_dir / f"{cache_key}.url"
+    url_file.write_text(url, encoding="utf-8")
+
+    logger.debug(f"Cached {url} to {cache_file}")
+
+
+def _read_from_cache(url: str, cache_dir: Path) -> str | None:
+    """Read cached HTTP content from disk.
+
+    Args:
+        url: The URL to look up in cache
+        cache_dir: Directory containing cache files
+
+    Returns:
+        Cached content if found, None otherwise
+
+    """
+    if not cache_dir.exists():
+        return None
+
+    cache_key = _get_cache_key(url)
+    cache_file = cache_dir / cache_key
+
+    if cache_file.exists():
+        logger.debug(f"Cache hit for {url} from {cache_file}")
+        return cache_file.read_text(encoding="utf-8")
+
+    return None
+
+
 def process_line(
     line: str,
     package_keys: list[str],
     override_keys: list[str],
     ignore_keys: list[str],
     variety: str,
+    offline: bool = False,
+    cache_dir: Path | None = None,
 ) -> tuple[list[str], list[str]]:
     """Take line from a constraints or requirements file and process it recursively.
 
@@ -41,6 +108,8 @@ def process_line(
             override_keys=override_keys,
             ignore_keys=ignore_keys,
             variety="c",
+            offline=offline,
+            cache_dir=cache_dir,
         )
     elif line.startswith("-r"):
         return resolve_dependencies(
@@ -49,6 +118,8 @@ def process_line(
             override_keys=override_keys,
             ignore_keys=ignore_keys,
             variety="r",
+            offline=offline,
+            cache_dir=cache_dir,
         )
     try:
         parsed = Requirement(line)
@@ -78,14 +149,18 @@ def process_io(
     override_keys: list[str],
     ignore_keys: list[str],
     variety: str,
+    offline: bool = False,
+    cache_dir: Path | None = None,
 ) -> None:
     """Read lines from an open file and trigger processing of each line
 
     each line is processed and the result appendend to given requirements
     and constraint lists.
     """
     for line in fio:
-        new_requirements, new_constraints = process_line(line, package_keys, override_keys, ignore_keys, variety)
+        new_requirements, new_constraints = process_line(
+            line, package_keys, override_keys, ignore_keys, variety, offline, cache_dir
+        )
         requirements += new_requirements
         constraints += new_constraints
 
@@ -96,10 +171,23 @@ def resolve_dependencies(
     override_keys: list[str],
     ignore_keys: list[str],
     variety: str = "r",
+    offline: bool = False,
+    cache_dir: Path | None = None,
 ) -> tuple[list[str], list[str]]:
     """Takes a file or url, loads it and trigger to recursivly processes its content.
 
-    returns tuple of requirements and constraints
+    Args:
+        file_or_url: Path to local file or HTTP(S) URL
+        package_keys: List of package names being developed from source
+        override_keys: List of package names with version overrides
+        ignore_keys: List of package names to ignore
+        variety: "r" for requirements, "c" for constraints
+        offline: If True, use cached HTTP content and don't make network requests
+        cache_dir: Directory for caching HTTP content (default: ./.mxdev_cache)
+
+    Returns:
+        Tuple of (requirements, constraints) as lists of strings
+
     """
     requirements: list[str] = []
     constraints: list[str] = []
@@ -113,6 +201,10 @@ def resolve_dependencies(
     # Windows drive letters are single characters, URL schemes are longer
     is_url = parsed.scheme and len(parsed.scheme) > 1
 
+    # Default cache directory
+    if cache_dir is None:
+        cache_dir = Path(".mxdev_cache")
+
     if not is_url:
         requirements_in_file = Path(file_or_url)
         if requirements_in_file.exists():
@@ -125,25 +217,51 @@ def resolve_dependencies(
                     override_keys,
                     ignore_keys,
                     variety,
+                    offline,
+                    cache_dir,
                 )
         else:
             logger.info(
                 f"Can not read {variety_verbose} file '{file_or_url}', " "it does not exist. Empty file assumed."
             )
     else:
-        try:
-            with request.urlopen(file_or_url) as fio:
-                process_io(
-                    fio,
-                    requirements,
-                    constraints,
-                    package_keys,
-                    override_keys,
-                    ignore_keys,
-                    variety,
+        # HTTP(S) URL handling with caching
+        content: str
+        if offline:
+            # Offline mode: try to read from cache
+            cached_content = _read_from_cache(file_or_url, cache_dir)
+            if cached_content is None:
+                raise RuntimeError(
+                    f"Offline mode: HTTP reference '{file_or_url}' not found in cache. "
+                    f"Run mxdev in online mode first to populate the cache at {cache_dir}"
                 )
-        except URLError as e:
-            raise Exception(f"Failed to fetch '{file_or_url}': {e}")
+            content = cached_content
+            logger.info(f"Using cached content for {file_or_url}")
+        else:
+            # Online mode: fetch from HTTP and cache it
+            try:
+                with request.urlopen(file_or_url) as fio:
+                    content = fio.read().decode("utf-8")
+                # Cache the content for future offline use
+                _cache_http_content(file_or_url, content, cache_dir)
+            except URLError as e:
+                raise Exception(f"Failed to fetch '{file_or_url}': {e}")
+
+        # Process the content (either from cache or fresh from HTTP)
+        from io import StringIO
+
+        with StringIO(content) as fio:
+            process_io(
+                fio,
+                requirements,
+                constraints,
+                package_keys,
+                override_keys,
+                ignore_keys,
+                variety,
+                offline,
+                cache_dir,
+            )
 
     if requirements and variety == "r":
         requirements = (
@@ -172,12 +290,16 @@ def read(state: State) -> None:
 
     The result is stored on the state object
     """
+    from .config import to_bool
+
     cfg = state.configuration
+    offline = to_bool(cfg.settings.get("offline", False))
     state.requirements, state.constraints = resolve_dependencies(
         file_or_url=cfg.infile,
         package_keys=cfg.package_keys,
         override_keys=cfg.override_keys,
         ignore_keys=cfg.ignore_keys,
+        offline=offline,
     )