diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index 74d0bf6..d099d0a 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -404,6 +404,24 @@ devbase plugin info devbase plugin sync ``` +### `devbase plugin migrate` + +旧形式 (`plugins/` へのコピー) でインストールされたプラグインを、`repos/` 配下の永続クローンへ移行します。`install` / `update` 実行時にも自動で呼び出されるため、通常は手動実行不要です。 + +``` +devbase plugin migrate +``` + +移行の挙動: + +| 状況 | 動作 | +|---|---| +| コピーがクローンと一致 | 旧コピーを削除し `repos/` へ移行 (migrated) | +| コピーにローカル変更あり | 旧コピーを `plugins/.bak` として保全 (preserved、手動で reconcile) | +| 移行できない (ソース未登録 等) | スキップしてエラーを表示 (skipped) | + +`--link` でインストールしたプラグインは移行対象外です。 + ### `devbase plugin repo add` プラグインリポジトリを登録します。 diff --git a/lib/devbase/cli.py b/lib/devbase/cli.py index 9cd1ce1..6512c2e 100644 --- a/lib/devbase/cli.py +++ b/lib/devbase/cli.py @@ -234,6 +234,9 @@ def _add_plugin_parser(subparsers): pl_sub.add_parser('sync', help='Resync project symlinks') + pl_sub.add_parser('migrate', + help='Migrate legacy plugins/ installs to repos/ clones') + # Plugin repo sub-subcommands pl_repo = pl_sub.add_parser('repo', help='Manage plugin repositories') pl_repo_sub = pl_repo.add_subparsers(dest='repo_command') diff --git a/lib/devbase/commands/plugin.py b/lib/devbase/commands/plugin.py index b20d31d..f6acf61 100644 --- a/lib/devbase/commands/plugin.py +++ b/lib/devbase/commands/plugin.py @@ -9,6 +9,7 @@ from devbase.plugin.updater import update_plugin from devbase.plugin.info import show_plugin_info, show_available_plugins from devbase.plugin.syncer import sync_projects +from devbase.plugin.migrator import migrate from devbase.plugin.repo_manager import ( add_repository, remove_repository, @@ -34,6 +35,7 @@ def cmd_plugin(devbase_root: Path, args) -> int: 'update': lambda: cmd_plugin_update(devbase_root, getattr(args, 'name', None)), 'info': lambda: cmd_plugin_info(devbase_root, getattr(args, 'name', '')), 'sync': lambda: cmd_sync(devbase_root), + 'migrate': lambda: cmd_plugin_migrate(devbase_root), 'repo': lambda: cmd_repo(devbase_root, args), } @@ -138,6 +140,36 @@ def cmd_sync(devbase_root: Path) -> int: return 0 +def cmd_plugin_migrate(devbase_root: Path) -> int: + """Migrate legacy plugins/ copy installs to repos/ persistent clones""" + registry = PluginRegistry(devbase_root) + try: + result = migrate(registry) + except DevbaseError as e: + logger.error("%s", e) + return 1 + + if not (result.migrated or result.preserved or result.skipped): + logger.info("No legacy plugins/ installs to migrate.") + return 0 + + if result.migrated: + logger.info("Migrated %d plugin(s) to repos/: %s", + len(result.migrated), ", ".join(result.migrated)) + if result.preserved: + logger.warning( + "Preserved %d plugin(s) with local changes as plugins/.bak " + "(reconcile manually): %s", + len(result.preserved), ", ".join(result.preserved)) + if result.skipped: + logger.warning("Could not migrate %d plugin(s): %s", + len(result.skipped), ", ".join(result.skipped)) + for err in result.errors: + logger.warning(" %s", err) + return 1 + return 0 + + def cmd_repo(devbase_root: Path, args) -> int: """Dispatch repo subcommands""" registry = PluginRegistry(devbase_root) diff --git a/lib/devbase/plugin/installer.py b/lib/devbase/plugin/installer.py index 747c548..96ffb6d 100644 --- a/lib/devbase/plugin/installer.py +++ b/lib/devbase/plugin/installer.py @@ -81,6 +81,32 @@ def resolve_repo_url(repo: str) -> str: return f"https://github.com/{repo}.git" +def _auto_migrate(registry: PluginRegistry) -> None: + """Migrate any legacy plugins/ copy installs to repos/ before proceeding. + + Triggered on the first install/update after upgrading to repos/-based + plugin management so users do not have to run `devbase plugin migrate` + manually. No-op when nothing legacy remains. + """ + from .migrator import migrate, needs_migration + if not needs_migration(registry): + return + logger.info("Legacy plugins/ installs detected — migrating to repos/...") + result = migrate(registry) + if result.migrated: + logger.info(" Migrated: %s", ", ".join(result.migrated)) + # preserved/skipped recur on every install/update until the user + # reconciles, so avoid re-emitting a loud per-plugin WARNING each time: + # surface a single concise hint pointing at the explicit command, which + # prints the full per-plugin detail when run. + if result.preserved or result.skipped: + pending = len(result.preserved) + len(result.skipped) + logger.info( + " %d plugin(s) still need attention — run 'devbase plugin migrate' " + "for details.", pending, + ) + + def install_plugin( registry: PluginRegistry, source_str: str, @@ -91,6 +117,8 @@ def install_plugin( Raises PluginError on failure. """ + _auto_migrate(registry) + source = PluginSource.parse(source_str, link=link) plugins_dir = registry.get_plugins_dir() diff --git a/lib/devbase/plugin/migrator.py b/lib/devbase/plugin/migrator.py new file mode 100644 index 0000000..bba767b --- /dev/null +++ b/lib/devbase/plugin/migrator.py @@ -0,0 +1,504 @@ +"""Plugin migrator - migrates legacy plugins/ copy installs to repos/ clones""" + +import re +import shutil +import stat +from dataclasses import dataclass, field +from pathlib import Path + +from devbase.errors import PluginError +from devbase.log import get_logger + +from .models import AvailablePlugin, InstalledPlugin, RegisteredRepository +from .registry import PluginRegistry + +logger = get_logger("devbase.plugin.migrator") + + +@dataclass +class MigrationResult: + """Outcome of a plugins/ -> repos/ migration run.""" + migrated: list[str] = field(default_factory=list) # cleanly moved + copy deleted + preserved: list[str] = field(default_factory=list) # copy differed -> kept as .bak + skipped: list[str] = field(default_factory=list) # could not migrate + errors: list[str] = field(default_factory=list) + plugins_dir_cleaned: bool = False # plugins/ emptied to .gitkeep + + +def _is_legacy_plugin(plugin: InstalledPlugin) -> bool: + """True if a plugin still uses the pre-PLAN04 plugins/ copy format. + + --link installs also live under plugins/ but are intentional and must not + be migrated, so they are excluded. + """ + if plugin.linked: + return False + parts = Path(plugin.path).parts + return len(parts) >= 1 and parts[0] == 'plugins' + + +def needs_migration(registry: PluginRegistry) -> bool: + """True if any installed plugin still uses the legacy plugins/ copy format.""" + return any(_is_legacy_plugin(p) for p in registry.list_installed()) + + +# Names produced by _unique_bak_path: ".bak" or ".bak-". +_BAK_NAME_RE = re.compile(r'\.bak(-\d+)?$') + + +def _is_bak_name(name: str) -> bool: + """True if name matches the preserved-copy convention (.bak[-N]). + + A substring check like ``'.bak' in name`` would wrongly flag unrelated + entries such as ``my.bakery`` or ``notes.bak.txt``; anchor to the suffix. + """ + return _BAK_NAME_RE.search(name) is not None + + +def _unique_bak_path(old_dir: Path) -> Path: + """Return a non-existing .bak path, suffixing -2, -3, … if needed. + + A previous migration run may have already preserved .bak awaiting + manual reconciliation; never overwrite it, so each diverged copy lands in + its own directory. + """ + bak = old_dir.with_name(old_dir.name + '.bak') + if not bak.exists(): + return bak + n = 2 + while True: + candidate = old_dir.with_name(f"{old_dir.name}.bak-{n}") + if not candidate.exists(): + return candidate + n += 1 + + +def _entry_kind(p: Path) -> str: + """Classify a path for diff purposes: symlink / dir / file / other. + + Symlinks are checked first (a symlink to a dir would otherwise read as a + dir) so that a copy/clone type mismatch is always detected. + """ + if p.is_symlink(): + return 'symlink' + if p.is_dir(): + return 'dir' + if p.is_file(): + return 'file' + return 'other' + + +_EXEC_BITS = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + + +def _files_equal(fa: Path, fb: Path) -> bool: + """Compare two regular files by exec bits, size, then byte content. + + Reads in fixed-size chunks rather than slurping the whole file so a large + plugin asset can't exhaust memory during the migration scan. Only the + execute bits are compared (not the full S_IMODE): a local exec-bit change + (e.g. an entry script the user made executable) is functionally meaningful + and must be preserved, but read/write permission differences caused by the + environment's umask or group settings should not spuriously force a .bak. + """ + sa, sb = fa.stat(), fb.stat() + if (sa.st_mode & _EXEC_BITS) != (sb.st_mode & _EXEC_BITS): + return False + if sa.st_size != sb.st_size: + return False + chunk = 64 * 1024 + with fa.open('rb') as a, fb.open('rb') as b: + while True: + ba, bb = a.read(chunk), b.read(chunk) + if ba != bb: + return False + if not ba: + return True + + +def _dirs_differ(copy_dir: Path, repo_dir: Path) -> bool: + """True if deleting the legacy copy would discard data not in the clone. + + Walks *every* entry (regular files, symlinks, and directories — including + empty ones). An entry is treated as divergence only when it represents + data the copy holds but the clone does not: a copy-only entry (user-added + file/symlink/empty dir), or a common entry whose type, symlink target, or + file content differs. Upstream-only additions (present in the clone but + not the copy) are *not* a difference — deleting the copy loses nothing — so + a routine upstream change no longer forces a manual-reconcile .bak. + """ + def _entries(base: Path) -> set[Path]: + return {p.relative_to(base) for p in base.rglob('*')} + + copy_entries = _entries(copy_dir) + repo_entries = _entries(repo_dir) + + # User-added entries live only in the copy and would be lost on delete. + if copy_entries - repo_entries: + return True + + for rel in copy_entries: + fa, fb = copy_dir / rel, repo_dir / rel + kind = _entry_kind(fa) + if kind != _entry_kind(fb): + return True + if kind == 'symlink': + if fa.readlink() != fb.readlink(): + return True + elif kind == 'file': + if not _files_equal(fa, fb): + return True + elif kind == 'other': + # A socket / pipe / device the copy holds can't be content-compared, + # so it can't be proven identical — treat it as divergence and fall + # back to preserving the copy (.bak) rather than risk deleting data + # we couldn't inspect. + return True + return False + + +def _clone_is_healthy(clone_dir: Path) -> bool: + """True if clone_dir looks like a usable repo clone (has .git + registry.yml). + + A repos/ dir that lost its .git (or whose registry.yml went missing) points + migrated plugins at a tree that can never be pulled/updated again, so it + must be re-cloned rather than reused. + """ + return ( + clone_dir.is_dir() + and (clone_dir / '.git').exists() + and (clone_dir / 'registry.yml').is_file() + ) + + +def _reclaim_or_protect_existing(clone_dir: Path) -> None: + """Clear a reusable leftover at clone_dir, or protect a .git-bearing tree. + + Called before (re-)cloning into clone_dir. Three cases: + + - clone_dir is a symlink (broken, to a file, or even to a dir) -> remove + the link; it is never a real persistent clone and git_clone would fail. + - clone_dir does not exist -> nothing to do. + - clone_dir exists but is not a directory (a stray file squatting on the + path) -> remove it so git_clone can create the directory; a regular file + cannot hold a git working tree, so nothing is lost. + - clone_dir is a directory without .git -> a broken/partial clone that can + never be pulled; remove it so a fresh clone repairs it. + - clone_dir is a directory *with* .git -> it may hold uncommitted or + unpushed local work, so refuse to delete it and raise asking the user to + repair/remove it manually rather than silently destroying their changes. + """ + # Check the symlink first: is_dir()/exists() both follow symlinks, so a + # symlink-to-dir would otherwise slip through as a "directory". + if clone_dir.is_symlink(): + clone_dir.unlink() + return + if not clone_dir.exists(): + return + if not clone_dir.is_dir(): + # A regular file is squatting on the path; git_clone would fail. It can + # hold no git working tree, so removing it loses nothing. + clone_dir.unlink() + return + if not (clone_dir / '.git').exists(): + shutil.rmtree(clone_dir) + return + raise PluginError( + f"Existing clone '{clone_dir}' has a .git but is missing " + f"registry.yml; refusing to delete it to avoid losing local " + f"changes. Restore registry.yml (e.g. 'git checkout -- " + f"registry.yml') or remove the directory manually, then retry." + ) + + +def _build_persisted_repo( + registry: PluginRegistry, repo: RegisteredRepository, dir_name: str, +) -> RegisteredRepository: + """Build a repo row with local_path = repos/ + a refreshed plugin + list, WITHOUT saving. + + Used after a fresh clone *and* when reusing a healthy clone left by an + earlier run (a pre-persistent-clone registration with no local_path), so the + plugins.yml entry is repaired identically in both cases and future runs take + the local_path fast path. + + The caller is responsible for persisting the returned row: during migration + every repo update is accumulated and flushed in a single plugins.yml save + (see migrate()), so this no longer writes per repo. + """ + from .installer import parse_registry_yml + + local_path = f"repos/{dir_name}" + clone_dir = registry.devbase_root / local_path + reg_info = parse_registry_yml(clone_dir) + plugins = [ + AvailablePlugin(name=e.name, description=e.description, path=e.path) + for e in reg_info.plugins + ] if reg_info else list(repo.plugins) + return RegisteredRepository( + name=repo.name, url=repo.url, added_at=repo.added_at, + local_path=local_path, plugins=plugins, + ) + + +def _ensure_repo_cloned( + registry: PluginRegistry, + repo: RegisteredRepository, + pending_repos: list[RegisteredRepository], +) -> tuple[Path, RegisteredRepository]: + """Return the repos/ clone dir for a repo, cloning it if necessary. + + If the repo was registered before persistent-clone support (no local_path), + the clone dir is missing, or the existing clone is broken (missing .git / + registry.yml), perform a full clone and stage local_path + a refreshed + plugin list for persistence. A healthy repos/ clone left by an + earlier run is reused (and its missing local_path staged) rather than + re-cloned or protected. + + Any repo row that needs (re)persisting is appended to `pending_repos` + instead of being saved here; migrate() flushes them in a single + plugins.yml save before any destructive cleanup, so the registry still + durably points at the clone before old copies are retired, but the save + count stays O(1) rather than one save per cloned repo. + """ + from .installer import git_clone, parse_registry_yml + from .repo_manager import _url_to_repos_dirname + + if repo.local_path: + clone_dir = registry.devbase_root / repo.local_path + if _clone_is_healthy(clone_dir): + return clone_dir, repo + _reclaim_or_protect_existing(clone_dir) + + dir_name = _url_to_repos_dirname(repo.url) + repos_dir = registry.get_repos_dir() + repos_dir.mkdir(exist_ok=True) + clone_dir = repos_dir / dir_name + + # A pre-persistent-clone registration (no local_path) may already have a + # healthy repos/ clone from an earlier run; reuse it instead of + # protecting (raising) on its .git, just like the local_path branch above — + # only stage the missing local_path so future runs take the fast path. + if _clone_is_healthy(clone_dir): + updated = _build_persisted_repo(registry, repo, dir_name) + pending_repos.append(updated) + return clone_dir, updated + + # A leftover from a previously interrupted clone (e.g. disk full or network + # drop) would otherwise be reused forever — re-cloning is skipped while + # parse_registry_yml() keeps failing, so migration can never self-heal. + # Remove a leftover that is *not* a valid clone (missing .git, or a stray + # non-directory squatting on the path) so the clone below re-creates it + # cleanly; but a dir that still has .git may hold uncommitted/unpushed work + # and is protected (raises) rather than destroyed. + _reclaim_or_protect_existing(clone_dir) + + freshly_cloned = False + if not clone_dir.is_dir(): + try: + git_clone(repo.url, clone_dir, shallow=False) + freshly_cloned = True + except Exception: + # Drop a partial clone so the next run starts from a clean slate. + if clone_dir.is_dir(): + shutil.rmtree(clone_dir) + raise + + reg_info = parse_registry_yml(clone_dir) + if not reg_info: + # registry.yml is missing/invalid. Only discard a clone we just made + # (guaranteed to hold no local work); an existing .git-bearing dir is + # protected so we never delete uncommitted/unpushed changes — surface a + # recoverable error instead, mirroring the local_path branch. + if freshly_cloned: + shutil.rmtree(clone_dir) + raise PluginError( + f"No registry.yml found in cloned repository '{repo.name}'." + ) + raise PluginError( + f"Existing clone '{clone_dir}' has a .git but is missing " + f"registry.yml; refusing to delete it to avoid losing local " + f"changes. Restore registry.yml (e.g. 'git checkout -- " + f"registry.yml') or remove the directory manually, then retry." + ) + + updated = _build_persisted_repo(registry, repo, dir_name) + pending_repos.append(updated) + logger.info("Repository '%s' cloned to %s", repo.name, updated.local_path) + return clone_dir, updated + + +def _cleanup_plugins_dir(registry: PluginRegistry) -> bool: + """Normalize plugins/ to just .gitkeep once it holds no live copy installs. + + Conservatively kept untouched when anything still depends on it: + --link installs, skipped legacy copies still referenced by plugins.yml, or + preserved .bak directories awaiting manual reconciliation. Returns + True only when plugins/ was reduced to .gitkeep. + """ + plugins_dir = registry.get_plugins_dir() + if not plugins_dir.is_dir(): + return False + + if any(p.linked for p in registry.list_installed()): + return False + + # A skipped legacy install still points into plugins/ — keep its files. + if needs_migration(registry): + return False + + entries = [e for e in plugins_dir.iterdir() if e.name != '.gitkeep'] + bak_dirs = [e for e in entries if _is_bak_name(e.name)] + if bak_dirs: + logger.info( + "plugins/ retained: %d preserved .bak dir(s) await manual reconciliation", + len(bak_dirs), + ) + return False + + # Anything left over that is neither .gitkeep nor a preserved .bak means + # plugins/ is not actually clean; leave it untouched and report uncleaned. + leftover = [e for e in entries if e not in bak_dirs] + if leftover: + logger.info( + "plugins/ retained: %d unexpected entr(y/ies) remain (%s)", + len(leftover), ", ".join(sorted(e.name for e in leftover)), + ) + return False + + gitkeep = plugins_dir / '.gitkeep' + if not gitkeep.exists(): + gitkeep.touch() + return True + + +def migrate(registry: PluginRegistry, *, run_sync: bool = True) -> MigrationResult: + """Migrate legacy plugins/ copy installs to repos/ clones. + + For each legacy plugin: ensure its source repo is cloned to repos/, rewrite + InstalledPlugin.path to the repos/ location, then delete the old copy when + byte-identical or preserve it as .bak when it diverged. Finally + re-sync project symlinks and empty plugins/ to .gitkeep when safe. + """ + from .installer import parse_registry_yml + from .syncer import load_plugin_info, sync_projects + + result = MigrationResult() + legacy = [p for p in registry.list_installed() if _is_legacy_plugin(p)] + if not legacy: + return result + + # Two-phase migration so a registry-save failure can never leave a copy + # deleted while plugins.yml still points at the stale plugins/ path: + # + # Phase 1 (no destructive fs ops): validate the repo/clone/entry and + # decide each copy's fate (delete vs preserve as .bak), collecting the + # cloned-repo rows in `pending_repos`, the repos/ path rewrites in + # `pending`, and the retire actions in `retire`. Cloning stages the + # repo row in `pending_repos` rather than saving per clone, so the save + # count stays O(1) regardless of how many repos are cloned. + # Persist: write every repo row + path rewrite in a single plugins.yml + # save (save_migration), flushing the staged clones BEFORE any cleanup. + # Phase 2 (destructive): only after plugins.yml is durably at repos/ do we + # delete/rename the old copies. + # + # Ordering rationale: if the save raises, no copy has been touched yet, so + # the registry stays legacy and the next run retries cleanly with the copies + # intact (recoverable). Conversely the validated repos/ clone is known good + # before we commit, so committing the rewrite first cannot strand a plugin + # on a missing tree; a stray copy left by a phase-2 hiccup is merely surfaced + # by _cleanup_plugins_dir, never silent data loss. + pending: list[InstalledPlugin] = [] + pending_repos: list[RegisteredRepository] = [] # cloned-repo rows to persist + retire: list[tuple[str, Path, Path]] = [] # (plugin_name, old_dir, repo_dir) + + for plugin in legacy: + try: + repo = registry.get_repository_by_url(plugin.source) + if not repo: + result.skipped.append(plugin.name) + result.errors.append( + f"{plugin.name}: source repository not registered " + f"({plugin.source or 'no source URL'})" + ) + continue + + clone_dir, repo = _ensure_repo_cloned(registry, repo, pending_repos) + + reg_info = parse_registry_yml(clone_dir) + entry = None + if reg_info: + entry = next( + (e for e in reg_info.plugins if e.name == plugin.name), None, + ) + if not entry: + result.skipped.append(plugin.name) + result.errors.append( + f"{plugin.name}: not found in registry.yml of '{repo.name}'" + ) + continue + + repo_plugin_dir = clone_dir / entry.path.rstrip('/') + if not repo_plugin_dir.is_dir(): + result.skipped.append(plugin.name) + result.errors.append( + f"{plugin.name}: plugin dir missing in clone: {repo_plugin_dir}" + ) + continue + + rel_path = str(repo_plugin_dir.relative_to(registry.devbase_root)) + info = load_plugin_info(repo_plugin_dir) + version = info.version if info else plugin.version + + old_dir = registry.devbase_root / plugin.path + pending.append(InstalledPlugin( + name=plugin.name, + version=version, + source=plugin.source, + installed_at=plugin.installed_at, + path=rel_path, + linked=False, + )) + retire.append((plugin.name, old_dir, repo_plugin_dir)) + except Exception as e: + result.skipped.append(plugin.name) + result.errors.append(f"{plugin.name}: {e}") + + # Persist every staged cloned-repo row AND validated path rewrite in a + # single save BEFORE retiring any copy. This both (a) keeps the registry + # durably pointing at the repos/ clones before destructive cleanup — the + # two-phase atomicity invariant — and (b) collapses what used to be one save + # per cloned repo plus the path-rewrite save into a single O(1) write. A + # failure here aborts with the copies untouched (recoverable). + registry.save_migration(pending_repos, pending) + + # Now that plugins.yml durably points at repos/, retire the old copies. + for name, old_dir, repo_plugin_dir in retire: + try: + if old_dir.is_dir() and not old_dir.is_symlink(): + if _dirs_differ(old_dir, repo_plugin_dir): + bak = _unique_bak_path(old_dir) + old_dir.rename(bak) + result.preserved.append(name) + logger.warning( + "Plugin '%s' had local changes — preserved at %s " + "(reconcile manually, then remove)", + name, bak, + ) + else: + shutil.rmtree(old_dir) + result.migrated.append(name) + else: + result.migrated.append(name) + except Exception as e: + # Registry is already at repos/ (valid clone), so the plugin works; + # a copy we failed to retire just lingers under plugins/ and is + # surfaced by _cleanup_plugins_dir rather than lost. + result.migrated.append(name) + result.errors.append(f"{name}: copy not retired: {e}") + + if run_sync: + sync_projects(registry) + + result.plugins_dir_cleaned = _cleanup_plugins_dir(registry) + return result diff --git a/lib/devbase/plugin/registry.py b/lib/devbase/plugin/registry.py index f852fcb..b5eb6ed 100644 --- a/lib/devbase/plugin/registry.py +++ b/lib/devbase/plugin/registry.py @@ -65,11 +65,79 @@ def get(self, name: str) -> Optional[InstalledPlugin]: def add(self, plugin: InstalledPlugin) -> None: """Add a plugin to the registry""" - data = self._load() + self.add_many([plugin]) + + @staticmethod + def _apply_plugins(data: dict, plugins: list[InstalledPlugin]) -> None: + """Upsert `plugins` into `data['installed_plugins']` (in place). + + Duplicate names within `plugins` are de-duplicated (last one wins) so a + caller passing the same plugin twice can't leave two conflicting entries + in plugins.yml. Does not touch disk — callers persist `data` once. + """ + if not plugins: + return + # Keep only the last entry per name; dict preserves insertion order so + # the surviving entries stay in the order they were last supplied. + unique = list({p.name: p for p in plugins}.values()) + names = {p.name for p in unique} data['installed_plugins'] = [ - p for p in data['installed_plugins'] if p['name'] != plugin.name + p for p in data['installed_plugins'] if p['name'] not in names + ] + data['installed_plugins'].extend(p.to_dict() for p in unique) + + @staticmethod + def _apply_repositories( + data: dict, repos: list[RegisteredRepository], + ) -> None: + """Upsert `repos` into `data['repositories']` (in place). + + Duplicate names (last one wins) are de-duplicated so accumulating the + same repo twice can't leave two conflicting rows. Does not touch disk. + """ + if not repos: + return + unique = list({r.name: r for r in repos}.values()) + names = {r.name for r in unique} + data['repositories'] = [ + r for r in data['repositories'] if r['name'] not in names ] - data['installed_plugins'].append(plugin.to_dict()) + data['repositories'].extend(r.to_dict() for r in unique) + + def add_many(self, plugins: list[InstalledPlugin]) -> None: + """Add/replace multiple plugins with a single load+save. + + Saving plugins.yml once for a batch avoids the repeated read+atomic + rewrite that calling add() per plugin would incur (e.g. during + migration of many legacy installs). + + Duplicate names within `plugins` are de-duplicated (last one wins) so a + caller passing the same plugin twice can't leave two conflicting entries + in plugins.yml.""" + if not plugins: + return + data = self._load() + self._apply_plugins(data, plugins) + self._save(data) + + def save_migration( + self, + repositories: list[RegisteredRepository], + plugins: list[InstalledPlugin], + ) -> None: + """Persist repository + plugin updates from a migration in ONE save. + + migrate() clones each legacy repo and rewrites each plugin's path; both + the refreshed repository rows (local_path + plugin list) and the plugin + path rewrites must land durably *before* any old copy is deleted. This + applies both sets of upserts to a single loaded snapshot and writes + plugins.yml exactly once, so the save count stays O(1) regardless of how + many repos were cloned (rather than one save per cloned repo).""" + if not repositories and not plugins: + return + data = self._load() + self._apply_repositories(data, repositories) + self._apply_plugins(data, plugins) self._save(data) def remove(self, name: str) -> bool: @@ -125,10 +193,7 @@ def get_repository_by_url(self, url: str) -> Optional[RegisteredRepository]: def add_repository(self, repo: RegisteredRepository) -> None: """Add or update a repository in the registry""" data = self._load() - data['repositories'] = [ - r for r in data['repositories'] if r['name'] != repo.name - ] - data['repositories'].append(repo.to_dict()) + self._apply_repositories(data, [repo]) self._save(data) def remove_repository(self, name: str) -> bool: diff --git a/lib/devbase/plugin/updater.py b/lib/devbase/plugin/updater.py index 4fdd163..d66d18a 100644 --- a/lib/devbase/plugin/updater.py +++ b/lib/devbase/plugin/updater.py @@ -189,6 +189,9 @@ def update_plugin(registry: PluginRegistry, name: Optional[str] = None) -> None: Raises PluginError on failure. """ + from .installer import _auto_migrate + _auto_migrate(registry) + installed = registry.list_installed() if not installed: logger.info("No plugins installed") diff --git a/tests/plugin/test_migrator.py b/tests/plugin/test_migrator.py new file mode 100644 index 0000000..a3fb2c0 --- /dev/null +++ b/tests/plugin/test_migrator.py @@ -0,0 +1,1054 @@ +"""Tests for PLAN04 PR2: legacy plugins/ -> repos/ migration""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import yaml +import pytest + +from devbase.errors import PluginError +from devbase.plugin.models import ( + AvailablePlugin, + InstalledPlugin, + RegisteredRepository, +) +from devbase.plugin.registry import PluginRegistry +from devbase.plugin.migrator import ( + _cleanup_plugins_dir, + _clone_is_healthy, + _dirs_differ, + _is_bak_name, + _is_legacy_plugin, + migrate, + needs_migration, +) + +URL = "https://github.com/testorg/testrepo.git" +DIRNAME = "github.com--testorg--testrepo" + + +def _make_repo_clone(devbase_root: Path, plugins: list[dict]) -> Path: + """Create repos// with .git, registry.yml and plugin dirs.""" + repo_dir = devbase_root / "repos" / DIRNAME + repo_dir.mkdir(parents=True, exist_ok=True) + (repo_dir / ".git").mkdir(exist_ok=True) + + entries = [] + for p in plugins: + pdir = repo_dir / p["path"] + pdir.mkdir(parents=True, exist_ok=True) + (pdir / "plugin.yml").write_text( + yaml.dump({"name": p["name"], "version": p.get("version", "1.0.0")}) + ) + for proj in p.get("projects", []): + proj_dir = pdir / "projects" / proj + proj_dir.mkdir(parents=True, exist_ok=True) + (proj_dir / "compose.yml").write_text("services: {}\n") + entries.append({"name": p["name"], "path": p["path"], "description": ""}) + + (repo_dir / "registry.yml").write_text( + yaml.dump({"name": "testrepo", "plugins": entries}) + ) + return repo_dir + + +def _register_repo(registry: PluginRegistry, plugins: list[dict], + local_path: str | None = f"repos/{DIRNAME}") -> None: + registry.add_repository(RegisteredRepository( + name="testrepo", url=URL, added_at=registry.now_iso(), + local_path=local_path or "", + plugins=[ + AvailablePlugin(name=p["name"], description="", path=p["path"]) + for p in plugins + ], + )) + + +def _make_legacy_copy(devbase_root: Path, name: str, plugin: dict, + extra: dict[str, str] | None = None) -> Path: + """Create plugins// mirroring the repo plugin dir (optionally diverged).""" + pdir = devbase_root / "plugins" / name + pdir.mkdir(parents=True, exist_ok=True) + (pdir / "plugin.yml").write_text( + yaml.dump({"name": plugin["name"], "version": plugin.get("version", "1.0.0")}) + ) + for proj in plugin.get("projects", []): + proj_dir = pdir / "projects" / proj + proj_dir.mkdir(parents=True, exist_ok=True) + (proj_dir / "compose.yml").write_text("services: {}\n") + for rel, content in (extra or {}).items(): + f = pdir / rel + f.parent.mkdir(parents=True, exist_ok=True) + f.write_text(content) + return pdir + + +# ── Fixtures ──────────────────────────────────────────────────── + + +@pytest.fixture +def devbase_root(tmp_path): + (tmp_path / "projects").mkdir() + return tmp_path + + +@pytest.fixture +def registry(devbase_root): + return PluginRegistry(devbase_root) + + +def _installed(name: str, path: str, linked: bool = False) -> InstalledPlugin: + return InstalledPlugin( + name=name, + version="1.0.0", + source="https://github.com/testorg/testrepo.git", + installed_at="2026-01-01T00:00:00+00:00", + path=path, + linked=linked, + ) + + +# ── detection ─────────────────────────────────────────────────── + + +class TestIsLegacyPlugin: + def test_copy_install_under_plugins_is_legacy(self): + assert _is_legacy_plugin(_installed("adminer", "plugins/adminer")) is True + + def test_repos_based_is_not_legacy(self): + plugin = _installed( + "adminer", "repos/github.com--testorg--testrepo/adminer", + ) + assert _is_legacy_plugin(plugin) is False + + def test_linked_under_plugins_is_not_legacy(self): + plugin = _installed("local", "plugins/local", linked=True) + assert _is_legacy_plugin(plugin) is False + + +class TestDirsDiffer: + def _write(self, base: Path, rel: str, content: str) -> None: + p = base / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + + def test_identical_dirs_do_not_differ(self, tmp_path): + a, b = tmp_path / "a", tmp_path / "b" + self._write(a, "plugin.yml", "name: x\n") + self._write(a, "projects/p/compose.yml", "services: {}\n") + self._write(b, "plugin.yml", "name: x\n") + self._write(b, "projects/p/compose.yml", "services: {}\n") + assert _dirs_differ(a, b) is False + + def test_changed_file_content_differs(self, tmp_path): + a, b = tmp_path / "a", tmp_path / "b" + self._write(a, "plugin.yml", "name: x\nversion: 9\n") + self._write(b, "plugin.yml", "name: x\n") + assert _dirs_differ(a, b) is True + + def test_same_size_different_content_differs(self, tmp_path): + # Same byte length but different content — exercises the streamed + # chunk comparison rather than the size fast-path. + a, b = tmp_path / "a", tmp_path / "b" + self._write(a, "plugin.yml", "name: aaaa\n") + self._write(b, "plugin.yml", "name: bbbb\n") + assert _dirs_differ(a, b) is True + + def test_extra_file_in_a_differs(self, tmp_path): + a, b = tmp_path / "a", tmp_path / "b" + self._write(a, "plugin.yml", "name: x\n") + self._write(a, "projects/p/.env", "SECRET=1\n") + self._write(b, "plugin.yml", "name: x\n") + assert _dirs_differ(a, b) is True + + def test_upstream_only_addition_does_not_differ(self, tmp_path): + # A file present only upstream (in the clone) is not data the legacy + # copy holds, so deleting the copy loses nothing — a routine upstream + # addition must NOT force a manual-reconcile .bak. + a, b = tmp_path / "a", tmp_path / "b" + self._write(a, "plugin.yml", "name: x\n") + self._write(b, "plugin.yml", "name: x\n") + self._write(b, "README.md", "new upstream doc\n") + assert _dirs_differ(a, b) is False + + def test_extra_symlink_in_copy_differs(self, tmp_path): + a, b = tmp_path / "a", tmp_path / "b" + self._write(a, "plugin.yml", "name: x\n") + self._write(b, "plugin.yml", "name: x\n") + # User-added symlink present only in the legacy copy + (a / "link").symlink_to("plugin.yml") + assert _dirs_differ(a, b) is True + + def test_empty_dir_only_in_copy_differs(self, tmp_path): + a, b = tmp_path / "a", tmp_path / "b" + self._write(a, "plugin.yml", "name: x\n") + self._write(b, "plugin.yml", "name: x\n") + # User-added empty directory present only in the legacy copy + (a / "logs").mkdir(parents=True) + assert _dirs_differ(a, b) is True + + def test_symlink_target_change_differs(self, tmp_path): + a, b = tmp_path / "a", tmp_path / "b" + self._write(a, "plugin.yml", "name: x\n") + self._write(b, "plugin.yml", "name: x\n") + (a / "link").symlink_to("a-target") + (b / "link").symlink_to("b-target") + assert _dirs_differ(a, b) is True + + def test_file_vs_symlink_type_mismatch_differs(self, tmp_path): + a, b = tmp_path / "a", tmp_path / "b" + self._write(a, "plugin.yml", "name: x\n") + self._write(b, "plugin.yml", "name: x\n") + # Same name, but a regular file in the copy vs a symlink in the clone + self._write(a, "shared", "data\n") + (b / "shared").symlink_to("plugin.yml") + assert _dirs_differ(a, b) is True + + def test_identical_symlinks_do_not_differ(self, tmp_path): + a, b = tmp_path / "a", tmp_path / "b" + self._write(a, "plugin.yml", "name: x\n") + self._write(b, "plugin.yml", "name: x\n") + (a / "link").symlink_to("plugin.yml") + (b / "link").symlink_to("plugin.yml") + assert _dirs_differ(a, b) is False + + def test_exec_bit_change_differs(self, tmp_path): + # Same name, size and bytes, but the copy made the script executable. + # Deleting the copy would lose that mode change, so it must count as + # divergence (preserved as .bak) rather than a clean migration. + a, b = tmp_path / "a", tmp_path / "b" + self._write(a, "entry.sh", "#!/bin/sh\necho hi\n") + self._write(b, "entry.sh", "#!/bin/sh\necho hi\n") + (b / "entry.sh").chmod(0o644) + (a / "entry.sh").chmod(0o755) + # Sanity: bytes identical, only mode differs. + assert (a / "entry.sh").read_bytes() == (b / "entry.sh").read_bytes() + assert _dirs_differ(a, b) is True + # And once the modes match, the copy is treated as identical again. + (a / "entry.sh").chmod(0o644) + assert _dirs_differ(a, b) is False + + +class TestIsBakName: + def test_plain_bak_matches(self): + assert _is_bak_name("carmo.bak") is True + + def test_numbered_bak_matches(self): + assert _is_bak_name("carmo.bak-2") is True + assert _is_bak_name("carmo.bak-17") is True + + def test_substring_bak_does_not_match(self): + # The previous `'.bak' in name` check wrongly flagged these. + assert _is_bak_name("my.bakery") is False + assert _is_bak_name("notes.bak.txt") is False + assert _is_bak_name("backup") is False + + +class TestCloneIsHealthy: + def test_valid_clone_is_healthy(self, devbase_root): + plugins = [{"name": "adminer", "path": "adminer"}] + clone = _make_repo_clone(devbase_root, plugins) + assert _clone_is_healthy(clone) is True + + def test_missing_git_is_unhealthy(self, devbase_root): + plugins = [{"name": "adminer", "path": "adminer"}] + clone = _make_repo_clone(devbase_root, plugins) + shutil_rmtree_git(clone) + assert _clone_is_healthy(clone) is False + + def test_missing_registry_yml_is_unhealthy(self, devbase_root): + plugins = [{"name": "adminer", "path": "adminer"}] + clone = _make_repo_clone(devbase_root, plugins) + (clone / "registry.yml").unlink() + assert _clone_is_healthy(clone) is False + + +def shutil_rmtree_git(clone: Path) -> None: + import shutil + shutil.rmtree(clone / ".git") + + +class TestNeedsMigration: + def test_true_when_legacy_present(self, registry): + registry.add(_installed("adminer", "plugins/adminer")) + assert needs_migration(registry) is True + + def test_false_when_only_repos_and_linked(self, registry): + registry.add(_installed( + "adminer", "repos/github.com--testorg--testrepo/adminer", + )) + registry.add(_installed("local", "plugins/local", linked=True)) + assert needs_migration(registry) is False + + def test_false_when_empty(self, registry): + assert needs_migration(registry) is False + + +# ── migrate() ─────────────────────────────────────────────────── + + +class TestMigrateClean: + def test_clean_migration_updates_path_and_deletes_copy(self, registry, devbase_root): + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + + result = migrate(registry) + + # path rewritten to repos/-based location + migrated_plugin = registry.get("adminer") + assert migrated_plugin.path == f"repos/{DIRNAME}/adminer" + # old copy removed + assert not (devbase_root / "plugins" / "adminer").exists() + # result bookkeeping + assert result.migrated == ["adminer"] + assert result.preserved == [] + assert result.errors == [] + + def test_clean_migration_creates_repos_symlink(self, registry, devbase_root): + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + + migrate(registry) + + link = devbase_root / "projects" / "adminer" + assert link.is_symlink() + target = (link.parent / link.readlink()).resolve() + assert target == (devbase_root / "repos" / DIRNAME / "adminer" / "projects" / "adminer").resolve() + + def test_clean_migration_empties_plugins_dir_to_gitkeep(self, registry, devbase_root): + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + + result = migrate(registry) + + plugins_dir = devbase_root / "plugins" + remaining = sorted(p.name for p in plugins_dir.iterdir()) + assert remaining == [".gitkeep"] + assert result.plugins_dir_cleaned is True + + +class TestMigrateWithLocalChanges: + def test_diverged_copy_preserved_as_bak(self, registry, devbase_root): + plugins = [{"name": "carmo", "path": "carmo", "projects": ["carmo"]}] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + # User added a local .env that does not exist upstream + _make_legacy_copy(devbase_root, "carmo", plugins[0], + extra={"projects/carmo/.env": "LOCAL=1\n"}) + registry.add(_installed("carmo", "plugins/carmo")) + + result = migrate(registry) + + assert result.preserved == ["carmo"] + assert result.migrated == [] + bak = devbase_root / "plugins" / "carmo.bak" + assert bak.is_dir() + assert (bak / "projects" / "carmo" / ".env").read_text() == "LOCAL=1\n" + # original copy path no longer present + assert not (devbase_root / "plugins" / "carmo").exists() + + def test_bak_retained_plugins_dir_not_cleaned(self, registry, devbase_root): + plugins = [{"name": "carmo", "path": "carmo", "projects": ["carmo"]}] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "carmo", plugins[0], + extra={"extra.txt": "x\n"}) + registry.add(_installed("carmo", "plugins/carmo")) + + result = migrate(registry) + + assert result.plugins_dir_cleaned is False + # path is still rewritten to repos/ even when copy is preserved + assert registry.get("carmo").path == f"repos/{DIRNAME}/carmo" + + def test_existing_bak_is_not_overwritten(self, registry, devbase_root): + plugins = [{"name": "carmo", "path": "carmo", "projects": ["carmo"]}] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "carmo", plugins[0], + extra={"projects/carmo/.env": "LOCAL=1\n"}) + registry.add(_installed("carmo", "plugins/carmo")) + # A previous migration run already preserved carmo.bak with its own data + prev_bak = devbase_root / "plugins" / "carmo.bak" + prev_bak.mkdir(parents=True) + (prev_bak / "old.txt").write_text("PREVIOUS\n") + + result = migrate(registry) + + assert result.preserved == ["carmo"] + # the old .bak survives untouched + assert (prev_bak / "old.txt").read_text() == "PREVIOUS\n" + # the new diverged copy lands in a distinct .bak-2 dir + new_bak = devbase_root / "plugins" / "carmo.bak-2" + assert new_bak.is_dir() + assert (new_bak / "projects" / "carmo" / ".env").read_text() == "LOCAL=1\n" + + +class TestMigrateClonesMissingRepo: + def test_repo_without_local_path_is_cloned(self, registry, devbase_root): + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + # Repo registered WITHOUT local_path (legacy registration) + _register_repo(registry, plugins, local_path=None) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + + def fake_clone(url, dest, **kwargs): + _make_repo_clone(dest.parent.parent, plugins) + + with patch("devbase.plugin.installer.git_clone", side_effect=fake_clone): + result = migrate(registry) + + assert result.migrated == ["adminer"] + repo = registry.get_repository_by_url(URL) + assert repo.local_path == f"repos/{DIRNAME}" + assert registry.get("adminer").path == f"repos/{DIRNAME}/adminer" + + +class TestMigrateKeepsLinked: + def test_linked_install_keeps_plugins_dir(self, registry, devbase_root): + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + # A separate --link install lives under plugins/ and must be preserved + (devbase_root / "plugins" / "locallink").mkdir(parents=True) + registry.add(_installed("locallink", "plugins/locallink", linked=True)) + + result = migrate(registry) + + assert result.migrated == ["adminer"] + assert result.plugins_dir_cleaned is False + assert (devbase_root / "plugins" / "locallink").is_dir() + + +class TestMigrateSkips: + def test_unregistered_source_is_skipped(self, registry, devbase_root): + _make_legacy_copy( + devbase_root, "orphan", + {"name": "orphan", "path": "orphan", "projects": ["orphan"]}, + ) + registry.add(_installed("orphan", "plugins/orphan")) + + result = migrate(registry) + + assert result.skipped == ["orphan"] + assert result.migrated == [] + # untouched: copy stays, path unchanged + assert (devbase_root / "plugins" / "orphan").is_dir() + assert registry.get("orphan").path == "plugins/orphan" + + +class TestCleanupPluginsDir: + def test_unexpected_leftover_keeps_plugins_dir_uncleaned(self, registry, devbase_root): + # plugins/ holds a stray entry that is neither .gitkeep nor a .bak + # (e.g. left behind by an external tool); cleanup must not claim it + # cleaned and must leave the entry in place. + plugins_dir = devbase_root / "plugins" + plugins_dir.mkdir() + (plugins_dir / "stray").mkdir() + + assert _cleanup_plugins_dir(registry) is False + assert (plugins_dir / "stray").is_dir() + + def test_bak_lookalike_is_not_treated_as_preserved(self, registry, devbase_root): + # An entry whose name merely contains ".bak" as a substring (e.g. + # "my.bakery") is NOT a preserved copy; it must be reported as an + # unexpected leftover rather than silently retained as a .bak. + plugins_dir = devbase_root / "plugins" + plugins_dir.mkdir() + (plugins_dir / "my.bakery").mkdir() + + assert _cleanup_plugins_dir(registry) is False + # Still present (cleanup never deletes leftovers) and not mistaken for + # a .bak dir. + assert (plugins_dir / "my.bakery").is_dir() + + def test_numbered_bak_dir_is_retained(self, registry, devbase_root): + # A real preserved copy from a prior run (carmo.bak-2) keeps plugins/ + # uncleaned just like carmo.bak does. + plugins_dir = devbase_root / "plugins" + plugins_dir.mkdir() + (plugins_dir / "carmo.bak-2").mkdir() + + assert _cleanup_plugins_dir(registry) is False + assert (plugins_dir / "carmo.bak-2").is_dir() + + +class TestMigratePartialCloneRecovery: + def test_partial_clone_without_git_is_recloned(self, registry, devbase_root): + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + # Repo registered without local_path so migrate() takes the clone path. + _register_repo(registry, plugins, local_path=None) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + # Leftover partial clone from a prior interrupted run: a directory with + # no .git and no registry.yml that previously caused an infinite loop. + partial = devbase_root / "repos" / DIRNAME + partial.mkdir(parents=True) + (partial / "junk.txt").write_text("partial\n") + + def fake_clone(url, dest, **kwargs): + _make_repo_clone(dest.parent.parent, plugins) + + with patch("devbase.plugin.installer.git_clone", side_effect=fake_clone) as mock: + result = migrate(registry) + + # The broken dir was removed and a fresh clone performed. + mock.assert_called_once() + assert result.migrated == ["adminer"] + assert registry.get("adminer").path == f"repos/{DIRNAME}/adminer" + assert not (devbase_root / "repos" / DIRNAME / "junk.txt").exists() + + def test_registered_local_path_without_git_is_recloned(self, registry, devbase_root): + # local_path is recorded (repo migrated before) but the clone lost its + # .git — e.g. an interrupted operation. Reusing it would leave the + # migrated plugin pointing at an un-pullable tree, so it must re-clone. + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _register_repo(registry, plugins) # local_path = repos/ + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + # Existing repos/ dir that is NOT a valid clone (no .git, no registry.yml). + broken = devbase_root / "repos" / DIRNAME + broken.mkdir(parents=True) + (broken / "leftover.txt").write_text("broken\n") + + def fake_clone(url, dest, **kwargs): + _make_repo_clone(dest.parent.parent, plugins) + + with patch("devbase.plugin.installer.git_clone", side_effect=fake_clone) as mock: + result = migrate(registry) + + mock.assert_called_once() + assert result.migrated == ["adminer"] + assert (devbase_root / "repos" / DIRNAME / ".git").exists() + assert not (devbase_root / "repos" / DIRNAME / "leftover.txt").exists() + + +class TestMigrateBatchesRegistryWrites: + def test_multiple_plugins_all_persisted_with_single_save(self, registry, devbase_root): + plugins = [ + {"name": "adminer", "path": "adminer", "projects": ["adminer"]}, + {"name": "carmo", "path": "carmo", "projects": ["carmo"]}, + {"name": "redis", "path": "redis", "projects": ["redis"]}, + ] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + for p in plugins: + _make_legacy_copy(devbase_root, p["name"], p) + registry.add(_installed(p["name"], f"plugins/{p['name']}")) + + with patch.object( + PluginRegistry, "_save", autospec=True, side_effect=PluginRegistry._save, + ) as save_spy: + result = migrate(registry) + + assert sorted(result.migrated) == ["adminer", "carmo", "redis"] + # All three path rewrites land in a single plugins.yml save rather than + # one save per plugin. + assert save_spy.call_count == 1 + for p in plugins: + assert registry.get(p["name"]).path == f"repos/{DIRNAME}/{p['name']}" + + def test_many_cloned_repos_persist_with_single_save(self, registry, devbase_root): + """O(1) saves even when every repo must be freshly cloned. + + Previously _ensure_repo_cloned saved plugins.yml once per cloned repo + (via add_repository), so migrating N repos cost N repo-saves + 1 + path-rewrite save. migrate() now stages every cloned repo row and + flushes them together with the path rewrites in one save_migration, so + the save count is O(1) regardless of repo count — while still persisting + the clones BEFORE any copy is retired (two-phase atomicity). + """ + from devbase.plugin.repo_manager import _url_to_repos_dirname + + repos = [ + ("https://github.com/testorg/repo-a.git", "alpha"), + ("https://github.com/testorg/repo-b.git", "beta"), + ("https://github.com/testorg/repo-c.git", "gamma"), + ] + dirnames = {url: _url_to_repos_dirname(url) for url, _ in repos} + + for url, pname in repos: + # Register each repo WITHOUT local_path so _ensure_repo_cloned takes + # the fresh-clone path (which used to save per repo). + registry.add_repository(RegisteredRepository( + name=pname, url=url, added_at=registry.now_iso(), local_path="", + plugins=[AvailablePlugin(name=pname, description="", path=pname)], + )) + # Legacy plugins/ copy + installed entry pointing at it. + pdir = devbase_root / "plugins" / pname + pdir.mkdir(parents=True, exist_ok=True) + (pdir / "plugin.yml").write_text(yaml.dump({"name": pname, "version": "1.0.0"})) + (pdir / "projects" / pname).mkdir(parents=True, exist_ok=True) + (pdir / "projects" / pname / "compose.yml").write_text("services: {}\n") + registry.add(InstalledPlugin( + name=pname, version="1.0.0", source=url, + installed_at="2026-01-01T00:00:00+00:00", + path=f"plugins/{pname}", linked=False, + )) + + def fake_clone(url, dest, **kwargs): + # Build a healthy clone identical (byte-for-byte) to the legacy copy + # so it is cleanly retired, proving the registry was persisted first. + pname = dest.name.split("--")[-1].replace("repo-", "") + # Map dest dirname back to the plugin name via the registered repos. + for u, p in repos: + if dest.name == dirnames[u]: + pname = p + break + dest.mkdir(parents=True, exist_ok=True) + (dest / ".git").mkdir(exist_ok=True) + pdir = dest / pname + pdir.mkdir(parents=True, exist_ok=True) + (pdir / "plugin.yml").write_text(yaml.dump({"name": pname, "version": "1.0.0"})) + (pdir / "projects" / pname).mkdir(parents=True, exist_ok=True) + (pdir / "projects" / pname / "compose.yml").write_text("services: {}\n") + (dest / "registry.yml").write_text(yaml.dump( + {"name": pname, "plugins": [{"name": pname, "path": pname, "description": ""}]} + )) + + with patch("devbase.plugin.installer.git_clone", side_effect=fake_clone): + with patch.object( + PluginRegistry, "_save", autospec=True, + side_effect=PluginRegistry._save, + ) as save_spy: + result = migrate(registry) + + assert sorted(result.migrated) == ["alpha", "beta", "gamma"] + # 3 cloned repos + 3 path rewrites, yet a single plugins.yml write. + assert save_spy.call_count == 1 + for url, pname in repos: + # Each repo row got its local_path persisted (registry durably points + # at the clone)... + repo = registry.get_repository_by_url(url) + assert repo.local_path == f"repos/{dirnames[url]}" + # ...and each plugin path was rewritten to the repos/ clone. + assert registry.get(pname).path == f"repos/{dirnames[url]}/{pname}" + # The legacy copy was retired only because the registry was persisted + # before Phase 2 (the clone is byte-identical, so it is deleted). + assert not (devbase_root / "plugins" / pname).exists() + + +class TestCmdPluginMigrate: + def test_command_runs_migration(self, devbase_root): + from devbase.commands.plugin import cmd_plugin_migrate + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _make_repo_clone(devbase_root, plugins) + registry = PluginRegistry(devbase_root) + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + + rc = cmd_plugin_migrate(devbase_root) + + assert rc == 0 + assert PluginRegistry(devbase_root).get("adminer").path == f"repos/{DIRNAME}/adminer" + + def test_command_noop_when_nothing_to_migrate(self, devbase_root): + from devbase.commands.plugin import cmd_plugin_migrate + rc = cmd_plugin_migrate(devbase_root) + assert rc == 0 + + +class TestAutoMigrateOnInstall: + def test_install_triggers_migration_of_legacy(self, registry, devbase_root): + plugins = [ + {"name": "adminer", "path": "adminer", "projects": ["adminer"]}, + {"name": "carmo", "path": "carmo", "projects": ["carmo"]}, + ] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + + from devbase.plugin.installer import install_plugin + install_plugin(registry, "carmo") + + # pre-existing legacy install migrated as a side effect + assert registry.get("adminer").path == f"repos/{DIRNAME}/adminer" + # the explicitly requested install also succeeds + assert registry.get("carmo").path == f"repos/{DIRNAME}/carmo" + + +class TestAutoMigrateWarningSuppression: + def test_preserved_does_not_emit_loud_per_plugin_warning( + self, registry, devbase_root, caplog, + ): + # A diverged legacy copy is preserved as .bak. _auto_migrate runs on + # every install/update; it must not re-emit a loud per-plugin WARNING + # each time — a concise INFO hint pointing at `devbase plugin migrate` + # is enough (the explicit command prints the full detail). + plugins = [ + {"name": "carmo", "path": "carmo", "projects": ["carmo"]}, + {"name": "redis", "path": "redis", "projects": ["redis"]}, + ] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + # carmo diverges (user-added file) -> will be preserved, not migrated. + _make_legacy_copy(devbase_root, "carmo", plugins[0], + extra={"projects/carmo/.env": "LOCAL=1\n"}) + registry.add(_installed("carmo", "plugins/carmo")) + + from devbase.plugin.installer import _auto_migrate + import logging + with caplog.at_level(logging.INFO, logger="devbase.plugin.installer"): + _auto_migrate(registry) + + installer_logs = [ + r for r in caplog.records + if r.name == "devbase.plugin.installer" + ] + # No WARNING-level record from the auto path. + assert all(r.levelno < logging.WARNING for r in installer_logs) + # The concise hint is present. + joined = " ".join(r.getMessage() for r in installer_logs) + assert "devbase plugin migrate" in joined + + +class TestAutoMigrateOnUpdate: + def test_update_triggers_migration_of_legacy(self, registry, devbase_root): + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + + from devbase.plugin.updater import update_plugin + with patch("devbase.plugin.updater._git_pull"): + update_plugin(registry) + + assert registry.get("adminer").path == f"repos/{DIRNAME}/adminer" + # only auto-migration removes the stale plugins/ copy; a plain pull + # would rewrite the path but leave the old copy on disk. + assert not (devbase_root / "plugins" / "adminer").exists() + + +class TestAddManyDedup: + def test_duplicate_names_keep_last(self, registry): + # add_many is a public batch API; passing the same name twice must not + # leave two conflicting entries in plugins.yml — the last wins. + first = _installed("adminer", "repos/x/adminer") + second = InstalledPlugin( + name="adminer", version="2.0.0", + source="https://github.com/testorg/testrepo.git", + installed_at="2026-02-02T00:00:00+00:00", + path="repos/y/adminer", linked=False, + ) + registry.add_many([first, second]) + installed = registry.list_installed() + assert [p.name for p in installed] == ["adminer"] + # Last entry won. + assert registry.get("adminer").path == "repos/y/adminer" + assert registry.get("adminer").version == "2.0.0" + + def test_duplicate_does_not_duplicate_existing(self, registry): + # An existing entry replaced by a batch containing duplicates of that + # name still results in exactly one row. + registry.add(_installed("adminer", "plugins/adminer")) + registry.add_many([ + _installed("adminer", "repos/a/adminer"), + _installed("adminer", "repos/b/adminer"), + ]) + installed = [p for p in registry.list_installed() if p.name == "adminer"] + assert len(installed) == 1 + assert installed[0].path == "repos/b/adminer" + + +class TestSaveMigration: + """save_migration upserts repos + plugins in a single load+save.""" + + def test_repos_and_plugins_applied_in_one_save(self, registry): + repo = RegisteredRepository( + name="testrepo", url=URL, added_at=registry.now_iso(), + local_path=f"repos/{DIRNAME}", + plugins=[AvailablePlugin(name="adminer", description="", path="adminer")], + ) + plugin = _installed("adminer", f"repos/{DIRNAME}/adminer") + + with patch.object( + PluginRegistry, "_save", autospec=True, side_effect=PluginRegistry._save, + ) as save_spy: + registry.save_migration([repo], [plugin]) + + assert save_spy.call_count == 1 + assert registry.get_repository_by_url(URL).local_path == f"repos/{DIRNAME}" + assert registry.get("adminer").path == f"repos/{DIRNAME}/adminer" + + def test_repo_upsert_replaces_existing_row(self, registry): + # A repo registered without local_path is replaced (not duplicated) by + # the migration row carrying local_path. + registry.add_repository(RegisteredRepository( + name="testrepo", url=URL, added_at=registry.now_iso(), local_path="", + )) + updated = RegisteredRepository( + name="testrepo", url=URL, added_at=registry.now_iso(), + local_path=f"repos/{DIRNAME}", + ) + registry.save_migration([updated], []) + + repos = registry.list_repositories() + assert len(repos) == 1 + assert repos[0].local_path == f"repos/{DIRNAME}" + + def test_empty_inputs_do_not_save(self, registry): + with patch.object(PluginRegistry, "_save", autospec=True) as save_spy: + registry.save_migration([], []) + save_spy.assert_not_called() + + +class TestFilesEqualExecBitOnly: + """_files_equal should only care about exec bits, not full perms.""" + + def test_rw_perm_diff_does_not_differ(self, tmp_path): + # Identical bytes, both non-executable, but differing read/write bits + # (e.g. from a different umask) must NOT count as divergence. + a, b = tmp_path / "a", tmp_path / "b" + a.mkdir(); b.mkdir() + (a / "f.txt").write_text("same\n") + (b / "f.txt").write_text("same\n") + (a / "f.txt").chmod(0o644) + (b / "f.txt").chmod(0o640) + assert _dirs_differ(a, b) is False + + def test_exec_bit_diff_still_differs(self, tmp_path): + # Functionally meaningful exec-bit change is still detected. + a, b = tmp_path / "a", tmp_path / "b" + a.mkdir(); b.mkdir() + (a / "f.sh").write_text("#!/bin/sh\n") + (b / "f.sh").write_text("#!/bin/sh\n") + (a / "f.sh").chmod(0o755) + (b / "f.sh").chmod(0o644) + assert _dirs_differ(a, b) is True + + +class TestEnsureRepoClonedProtectsGit: + """A recorded local_path whose dir keeps .git must not be deleted.""" + + def test_local_path_with_git_missing_registry_is_not_deleted( + self, registry, devbase_root, + ): + # local_path recorded; clone has .git but registry.yml is gone. The dir + # may hold uncommitted/unpushed local work, so migration must refuse to + # rmtree it and raise instead of silently destroying it. + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _register_repo(registry, plugins) # local_path = repos/ + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + clone = devbase_root / "repos" / DIRNAME + clone.mkdir(parents=True) + (clone / ".git").mkdir() + (clone / "local-work.txt").write_text("uncommitted\n") + # No registry.yml -> _clone_is_healthy is False. + + def fake_clone(url, dest, **kwargs): # pragma: no cover - must not run + raise AssertionError("re-clone must not happen when .git is present") + + with patch("devbase.plugin.installer.git_clone", side_effect=fake_clone): + result = migrate(registry) + + # The plugin was skipped (not migrated) and the dir survives intact. + assert "adminer" in result.skipped + assert (clone / ".git").is_dir() + assert (clone / "local-work.txt").read_text() == "uncommitted\n" + # registry entry still points at the legacy path so it retries later. + assert registry.get("adminer").path == "plugins/adminer" + + def test_local_path_without_git_is_still_recloned(self, registry, devbase_root): + # Sanity: when .git is gone the dir is genuinely broken and re-cloning + # (the existing behaviour) still applies. + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + broken = devbase_root / "repos" / DIRNAME + broken.mkdir(parents=True) + (broken / "leftover.txt").write_text("broken\n") + + def fake_clone(url, dest, **kwargs): + _make_repo_clone(dest.parent.parent, plugins) + + with patch( + "devbase.plugin.installer.git_clone", side_effect=fake_clone, + ) as mock: + result = migrate(registry) + + mock.assert_called_once() + assert result.migrated == ["adminer"] + assert (devbase_root / "repos" / DIRNAME / ".git").exists() + assert not (devbase_root / "repos" / DIRNAME / "leftover.txt").exists() + + def test_derived_path_with_git_missing_registry_is_not_deleted( + self, registry, devbase_root, + ): + # local_path is NOT recorded (pre-persistent-clone registration) but a + # repos/ clone already exists with .git and only registry.yml + # missing. It may hold uncommitted/unpushed local work, so the derived + # clone path must refuse to rmtree it just as the local_path branch does. + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _register_repo(registry, plugins, local_path=None) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + clone = devbase_root / "repos" / DIRNAME + clone.mkdir(parents=True) + (clone / ".git").mkdir() + (clone / "local-work.txt").write_text("uncommitted\n") + # No registry.yml -> parse_registry_yml fails on the existing dir. + + def fake_clone(url, dest, **kwargs): # pragma: no cover - must not run + raise AssertionError("re-clone must not happen when .git is present") + + with patch("devbase.plugin.installer.git_clone", side_effect=fake_clone): + result = migrate(registry) + + assert "adminer" in result.skipped + assert (clone / ".git").is_dir() + assert (clone / "local-work.txt").read_text() == "uncommitted\n" + # registry entry still legacy so a later run can retry. + assert registry.get("adminer").path == "plugins/adminer" + + def test_derived_path_with_healthy_clone_is_reused( + self, registry, devbase_root, + ): + # local_path is NOT recorded (pre-persistent-clone registration) but a + # *healthy* repos/ clone (.git AND registry.yml) was left by an + # earlier run. It must be reused — migration proceeds, the missing + # local_path is persisted — not protected (skipped) on its .git nor + # re-cloned. + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _register_repo(registry, plugins, local_path=None) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + # Healthy clone already present (sentinel proves it is reused, not wiped). + clone = _make_repo_clone(devbase_root, plugins) + (clone / ".git" / "sentinel").write_text("kept\n") + + def fake_clone(url, dest, **kwargs): # pragma: no cover - must not run + raise AssertionError("must reuse a healthy clone, never re-clone") + + with patch("devbase.plugin.installer.git_clone", side_effect=fake_clone): + result = migrate(registry) + + # Migration succeeded (NOT skipped) and the existing clone survived. + assert result.migrated == ["adminer"] + assert "adminer" not in result.skipped + assert (clone / ".git" / "sentinel").read_text() == "kept\n" + # The previously-missing local_path was persisted, and the plugin now + # points at the reused repos/ clone. + repo = registry.get_repository_by_url(URL) + assert repo.local_path == f"repos/{DIRNAME}" + assert registry.get("adminer").path == f"repos/{DIRNAME}/adminer" + + def test_clone_dir_existing_as_file_is_replaced(self, registry, devbase_root): + # repos/ is squatted on by a regular *file* (not a directory). + # git_clone would fail; the file holds no git tree so it is removed and + # a fresh clone created in its place. + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _register_repo(registry, plugins, local_path=None) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + repos_dir = devbase_root / "repos" + repos_dir.mkdir(parents=True) + stray = repos_dir / DIRNAME + stray.write_text("not a directory\n") + assert stray.is_file() + + def fake_clone(url, dest, **kwargs): + _make_repo_clone(dest.parent.parent, plugins) + + with patch( + "devbase.plugin.installer.git_clone", side_effect=fake_clone, + ) as mock: + result = migrate(registry) + + mock.assert_called_once() + assert result.migrated == ["adminer"] + assert (devbase_root / "repos" / DIRNAME).is_dir() + assert (devbase_root / "repos" / DIRNAME / ".git").exists() + + +class TestMigratePersistsRegistryBeforeRetiringCopy: + """A registry-save failure must not leave a copy deleted with a stale path. + + Round 3 batched the path rewrites into a single save AFTER retiring the + copies; if that save raised, the copies were already gone/renamed while + plugins.yml still pointed at plugins/. migrate() now persists the rewrites + (and any freshly cloned repo rows) via a single save_migration call BEFORE + any destructive filesystem op, so a save failure aborts with every copy + intact. + """ + + def test_save_failure_leaves_copy_intact(self, registry, devbase_root): + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + + # save_migration blows up exactly when migrate() tries to persist. + with patch.object( + PluginRegistry, "save_migration", side_effect=OSError("disk full"), + ): + with pytest.raises(OSError): + migrate(registry) + + # The copy was NOT deleted or renamed to .bak: it is still right where + # it was, so the next run can retry cleanly. + assert (devbase_root / "plugins" / "adminer" / "plugin.yml").is_file() + assert not (devbase_root / "plugins" / "adminer.bak").exists() + + def test_copy_retired_only_after_registry_persisted(self, registry, devbase_root): + # The copy delete/rename must happen strictly after save_migration returns. + plugins = [{"name": "adminer", "path": "adminer", "projects": ["adminer"]}] + _make_repo_clone(devbase_root, plugins) + _register_repo(registry, plugins) + _make_legacy_copy(devbase_root, "adminer", plugins[0]) + registry.add(_installed("adminer", "plugins/adminer")) + + copy = devbase_root / "plugins" / "adminer" + orig_save_migration = PluginRegistry.save_migration + + def spy_save_migration(self, repos_arg, plugins_arg): + # At save time the copy must still exist (not yet retired). + assert copy.is_dir(), "copy retired before registry was persisted" + return orig_save_migration(self, repos_arg, plugins_arg) + + with patch.object(PluginRegistry, "save_migration", autospec=True, + side_effect=spy_save_migration): + result = migrate(registry) + + assert result.migrated == ["adminer"] + # After a successful save the clean copy is gone. + assert not copy.exists() + assert registry.get("adminer").path == f"repos/{DIRNAME}/adminer" + + +class TestDirsDifferOtherEntryKind: + """An entry of kind 'other' (fifo/socket/device) can't be content-compared + and must be treated as divergence so the copy is preserved, not deleted.""" + + def test_fifo_in_copy_differs(self, tmp_path): + import os + a, b = tmp_path / "a", tmp_path / "b" + a.mkdir(); b.mkdir() + (a / "plugin.yml").write_text("name: x\n") + (b / "plugin.yml").write_text("name: x\n") + try: + os.mkfifo(a / "pipe") + os.mkfifo(b / "pipe") + except (AttributeError, NotImplementedError, OSError): + pytest.skip("mkfifo not supported on this platform") + # Both sides hold a fifo at the same path; it can't be proven identical + # so deleting the copy is unsafe -> divergence. + assert _dirs_differ(a, b) is True