Remove legacy importer and related legacy code (#738)

Author: thomashoneyman

AGENTS.md

@@ -84,7 +84,7 @@ nixfmt *.nix nix/**/*.nix
 - `app-e2e/` — E2E tests for the server API.
 - `lib/` — **Public library** for consumers (Spago, Pursuit, etc.). Only types and functions useful to external tools belong here. Avoid implementation-specific code.
 - `foreign/` — FFI bindings to JavaScript libraries.
-- `scripts/` — Runnable modules for registry tasks (LegacyImporter, PackageTransferrer, PackageSetUpdater, etc.). Run via `nix run .#legacy-importer`, etc.
+- `scripts/` — Runnable modules for registry tasks (PackageTransferrer, PackageSetUpdater, DailyImporter, etc.). Run via `nix run .#package-transferrer`, etc.
 - `test-utils/` — Shared test utilities.
 - `db/` — SQLite schemas and migrations (use `dbmate up` to initialize).
 - `types/` — Dhall type specifications.

CONTRIBUTING.md

@@ -127,11 +127,11 @@ nix run .#server
 You can also run any of the modules listed in the [scripts](./scripts/) directory by converting the camel-case file name to kebab-case, such as:
 
 ```sh
-# To run `LegacyImporter.purs`
-nix run .#legacy-importer
-
 # To run `PackageTransferrer.purs`
 nix run .#package-transferrer
+
+# To run `DailyImporter.purs`
+nix run .#daily-importer
 ```
 
 ### Required Environment Variables

SPEC.md

@@ -6,9 +6,9 @@ This specification defines the data formats used by the registry, the operations
 
 ## 1. Introduction
 
-The PureScript Registry is an independent package registry for PureScript. It was created after the Bower registry (which PureScript previously relied on for package distribution) stopped accepting new submissions in 2020. Initially package names were tracked in a simple `new-packages.json` file so that package managers like Spago and package sets tooling could continue to function. The full registry replaced this stopgap with a complete system.
+The PureScript Registry is an independent package registry for PureScript. It was created after the Bower registry (which PureScript previously relied on for package distribution) stopped accepting new submissions in 2020.
 
-Packages from the Bower registry and `new-packages.json` file have been grandfathered into the registry so long as they satisfy registry requirements. The most restrictive requirement is that package versions compile with at least one compiler version from 0.13.0 onwards. This means packages published prior to May 2019 are not in the registry and can only be used with legacy package managers (Bower, legacy Spago).
+Packages from the Bower registry have been grandfathered into the registry so long as they satisfy registry requirements. The most restrictive requirement is that package versions compile with at least one compiler version from 0.13.0 onwards. This means packages published prior to May 2019 are not in the registry and can only be used with legacy package managers (Bower, legacy Spago).
 
 The registry addresses several limitations of Bower and introduces new features of its own:
 

app-e2e/src/Test/E2E/Endpoint/Publish.purs

@@ -68,6 +68,28 @@ spec = do
           let actualCompilers = Set.fromFoldable $ NEA.toArray publishedMetaAfter.compilers
           Assert.shouldEqual actualCompilers expectedCompilers
 
+  Spec.describe "Publish with spago.dhall manifest" do
+    Spec.it "can publish slug@3.0.0 which uses spago.dhall format" do
+      { jobId } <- Client.publish Fixtures.slugPublishData
+      job <- Env.pollJobOrFail jobId
+      Assert.shouldSatisfy (V1.jobInfo job).finishedAt isJust
+
+      uploadOccurred <- Env.hasStorageUpload Fixtures.slug
+      unless uploadOccurred do
+        storageRequests <- WireMock.getStorageRequests
+        WireMock.failWithRequests "Expected S3 PUT for slug/3.0.0.tar.gz" storageRequests
+
+      Metadata metadata <- Env.readMetadata Fixtures.slug.name
+      case Map.lookup Fixtures.slug.version metadata.published of
+        Nothing -> Assert.fail $ "Expected version " <> Version.print Fixtures.slug.version <> " in metadata published versions"
+        Just publishedMeta -> do
+          Assert.shouldSatisfy (Sha256.print publishedMeta.hash) (not <<< String.null)
+
+      manifestEntries <- Env.readManifestIndexEntry Fixtures.slug.name
+      let hasVersion = Array.any (\(Manifest m) -> m.version == Fixtures.slug.version) manifestEntries
+      unless hasVersion do
+        Assert.fail $ "Expected version " <> Version.print Fixtures.slug.version <> " in manifest index"
+
   Spec.describe "Publish state machine" do
     Spec.it "returns same jobId for duplicate publish requests" do
       { jobId: id1 } <- Client.publish Fixtures.effectPublishData

app-e2e/src/Test/E2E/Support/Fixtures.purs

@@ -5,10 +5,12 @@ module Test.E2E.Support.Fixtures
   , effect
   , console
   , prelude
+  , slug
   , unsafeCoerce
   , effectPublishData
   , effectPublishDataDifferentLocation
   , consolePublishData
+  , slugPublishData
   , unsafeCoercePublishData
   , failingTransferData
   , nonexistentTransferData
@@ -53,6 +55,10 @@ console = { name: Utils.unsafePackageName "console", version: Utils.unsafeVersio
 prelude :: PackageFixture
 prelude = { name: Utils.unsafePackageName "prelude", version: Utils.unsafeVersion "6.0.1" }
 
+-- | slug@3.0.0 fixture package (uses spago.dhall manifest format)
+slug :: PackageFixture
+slug = { name: Utils.unsafePackageName "slug", version: Utils.unsafeVersion "3.0.0" }
+
 -- | Standard publish data for effect@4.0.0, used by E2E tests.
 -- | This matches the fixtures in app/fixtures/github-packages/effect-4.0.0
 effectPublishData :: Operation.PublishData
@@ -101,6 +107,23 @@ consolePublishData =
   , version: console.version
   }
 
+-- | Publish data for slug@3.0.0, used for spago.dhall manifest format test.
+-- | Slug depends on prelude only and uses spago.dhall as its manifest format.
+-- | This matches the fixtures in app/fixtures/github-packages/slug-3.0.0
+slugPublishData :: Operation.PublishData
+slugPublishData =
+  { name: slug.name
+  , location: Just $ GitHub
+      { owner: "purescript"
+      , repo: "purescript-slug"
+      , subdir: Nothing
+      }
+  , ref: "v3.0.0"
+  , compiler: Just $ Utils.unsafeVersion "0.15.10"
+  , resolutions: Nothing
+  , version: slug.version
+  }
+
 -- | Publish data for unsafe-coerce@6.0.0, used by package set tests.
 -- | Has no dependencies. Published first to create the tarball before adding to package set.
 unsafeCoercePublishData :: Operation.PublishData

app/fixtures/github-packages/slug-3.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Thomas Honeyman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

app/fixtures/github-packages/slug-3.0.0/packages.dhall

@@ -0,0 +1,7 @@
+-- Minimal packages.dhall for testing - defines only required dependencies inline
+{ prelude =
+  { dependencies = [] : List Text
+  , repo = "https://github.com/purescript/purescript-prelude.git"
+  , version = "v6.0.1"
+  }
+}

app/fixtures/github-packages/slug-3.0.0/spago.dhall

@@ -0,0 +1,7 @@
+{ name = "slug"
+, license = "MIT"
+, repository = "https://github.com/purescript/purescript-slug"
+, dependencies = [ "prelude" ]
+, packages = ./packages.dhall
+, sources = [ "src/**/*.purs" ]
+}

app/fixtures/github-packages/slug-3.0.0/src/Slug.purs

@@ -0,0 +1,13 @@
+module Slug where
+
+import Prelude
+
+-- | A URL-safe slug.
+newtype Slug = Slug String
+
+derive instance eqSlug :: Eq Slug
+derive instance ordSlug :: Ord Slug
+
+-- | Convert a Slug to a String.
+toString :: Slug -> String
+toString (Slug s) = s