diff --git a/hackage-server.cabal b/hackage-server.cabal index 0b3a43bb5..02ebb6aa6 100644 --- a/hackage-server.cabal +++ b/hackage-server.cabal @@ -149,7 +149,7 @@ common defaults , array >= 0.5 && < 0.6 , base >= 4.18 && < 4.22 , binary >= 0.8 && < 0.9 - , bytestring >= 0.10 && < 0.13 + , bytestring >= 0.11.2 && < 0.13 , containers >= 0.6.0 && < 0.9 , deepseq >= 1.4 && < 1.6 , directory >= 1.3 && < 1.4 diff --git a/src/Distribution/Server/Features/Core.hs b/src/Distribution/Server/Features/Core.hs index 0e9a7a941..1a506c2f1 100644 --- a/src/Distribution/Server/Features/Core.hs +++ b/src/Distribution/Server/Features/Core.hs @@ -27,7 +27,7 @@ import qualified Codec.Compression.GZip as GZip import Data.Aeson (Value (..), toJSON) import qualified Data.Aeson.Key as Key import qualified Data.Aeson.KeyMap as KeyMap -import Data.ByteString.Lazy (ByteString) +import Data.ByteString.Lazy (LazyByteString, fromStrict) import qualified Data.Foldable as Foldable import qualified Data.Text as Text import Data.Time.Clock (UTCTime, getCurrentTime) @@ -130,7 +130,7 @@ data CoreFeature = CoreFeature { -- modification time for the tar entry. -- -- This runs a `PackageChangeIndexExtra` hook when done. - updateArchiveIndexEntry :: forall m. MonadIO m => FilePath -> ByteString -> UTCTime -> m (), + updateArchiveIndexEntry :: forall m. MonadIO m => FilePath -> LazyByteString -> UTCTime -> m (), -- | Notification of package or index changes. packageChangeHook :: Hook PackageChange (), @@ -175,7 +175,7 @@ data PackageChange | PackageChangeInfo PackageUpdate PkgInfo PkgInfo -- | A file has changed in the package index tar not covered by any of the -- other change types. - | PackageChangeIndexExtra String ByteString UTCTime + | PackageChangeIndexExtra String LazyByteString UTCTime -- | A predicate to use with `packageChangeHook` and `registerHookJust` for -- keeping other features synchronized with the main package index. @@ -212,7 +212,7 @@ isPackageDeleteVersion :: Maybe PackageId, isPackageChangeCabalFile :: Maybe (PackageId, CabalFileText), isPackageChangeCabalFileUploadInfo :: Maybe (PackageId, UploadInfo), isPackageChangeTarball :: Maybe (PackageId, PkgTarball), -isPackageIndexExtraChange :: Maybe (String, ByteString, UTCTime) +isPackageIndexExtraChange :: Maybe (String, LazyByteString, UTCTime) -} data CoreResource = CoreResource { @@ -591,7 +591,7 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} runHook_ packageChangeHook (PackageChangeInfo PackageUpdatedUploadTime oldpkginfo newpkginfo) return True - updateArchiveIndexEntry :: MonadIO m => FilePath -> ByteString -> UTCTime -> m () + updateArchiveIndexEntry :: MonadIO m => FilePath -> LazyByteString -> UTCTime -> m () updateArchiveIndexEntry entryName entryData entryTime = logTiming maxBound ("updateArchiveIndexEntry " ++ show entryName) $ do updateState packagesState $ AddOtherIndexEntry $ ExtraEntry entryName entryData entryTime @@ -721,7 +721,7 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} -- check that the cabal name matches the package guard (lookup "cabal" dpath == Just (display $ packageName pkginfo)) let (fileRev, (utime, _uid)) = pkgLatestRevision pkginfo - cabalfile = Resource.CabalFile (cabalFileByteString fileRev) utime + cabalfile = Resource.CabalFile (fromStrict $ cabalFileByteString fileRev) utime return $ toResponse cabalfile serveCabalFileRevisionsList :: DynamicPath -> ServerPartE Response @@ -731,7 +731,7 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} let revisions = pkgMetadataRevisions pkginfo revisionToObj rev (cabalFileText, (utime, uid)) = let uname = userIdToName users uid - hash = sha256 (cabalFileByteString cabalFileText) + hash = sha256 (fromStrict $ cabalFileByteString cabalFileText) in Object $ KeyMap.fromList [ (Key.fromString "number", Number (fromIntegral rev)) @@ -750,7 +750,7 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} case mrev >>= \rev -> revisions Vec.!? rev of Just (fileRev, (utime, _uid)) -> return $ toResponse cabalfile where - cabalfile = Resource.CabalFile (cabalFileByteString fileRev) utime + cabalfile = Resource.CabalFile (fromStrict $ cabalFileByteString fileRev) utime Nothing -> errNotFound "Package revision not found" [MText "Cannot parse revision, or revision out of range."] diff --git a/src/Distribution/Server/Features/Core/Backup.hs b/src/Distribution/Server/Features/Core/Backup.hs index bc620a3d9..0fc1b6c45 100644 --- a/src/Distribution/Server/Features/Core/Backup.hs +++ b/src/Distribution/Server/Features/Core/Backup.hs @@ -98,7 +98,7 @@ doPackageImport (PartialIndex packages updatelog) entry = case entry of list <- importCSV "tarball.csv" bs >>= importTarballMetadata fp return $ partial { partialTarballUpload = list } [other] | Just version <- extractVersion other (packageName pkgId) ".cabal" -> - return $ partial { partialCabal = (version, CabalFileText bs):partialCabal partial } + return $ partial { partialCabal = (version, CabalFileText $ BS.toStrict bs) : partialCabal partial } _ -> return partial return $! PartialIndex (Map.insert pkgId partial' packages) updatelog BackupBlob filename@["package",pkgStr,other] blobId -> do @@ -198,7 +198,7 @@ partialToFullPkg (pkgId, PartialPkg{..}) = do filename = display pkgId ++ ".cabal" case runParseResult $ parseGenericPackageDescription $ - BS.toStrict $ cabalFileByteString latestCabalFile of + cabalFileByteString latestCabalFile of (_, Left (_, errs)) -> fail $ unlines (map (showPError filename) $ toList errs) (_, Right _) -> return () @@ -322,8 +322,8 @@ cabalListToExport pkgId cabalInfos = cabalName = display (packageName pkgId) ++ ".cabal" blobEntry :: (Int, CabalFileText) -> BackupEntry - blobEntry (0, CabalFileText bs) = BackupByteString (pkgPath pkgId cabalName) bs - blobEntry (n, CabalFileText bs) = BackupByteString (pkgPath pkgId (cabalName ++ "-" ++ show n)) bs + blobEntry (0, CabalFileText bs) = BackupByteString (pkgPath pkgId cabalName) (BS.fromStrict bs) + blobEntry (n, CabalFileText bs) = BackupByteString (pkgPath pkgId (cabalName ++ "-" ++ show n)) (BS.fromStrict bs) cabalMetadata :: CSV cabalMetadata = diff --git a/src/Distribution/Server/Features/EditCabalFiles.hs b/src/Distribution/Server/Features/EditCabalFiles.hs index 6b1154024..6161ccf47 100644 --- a/src/Distribution/Server/Features/EditCabalFiles.hs +++ b/src/Distribution/Server/Features/EditCabalFiles.hs @@ -23,7 +23,8 @@ import Distribution.Server.Util.CabalRevisions (Change(..), diffCabalRevisions, insertRevisionField) import Text.StringTemplate.Classes (SElem(SM)) -import Data.ByteString.Lazy (ByteString) +import Data.ByteString (StrictByteString) +import Data.ByteString.Lazy (LazyByteString) import qualified Data.ByteString.Lazy as BS.L import qualified Data.Map as Map import Data.Time (getCurrentTime) @@ -84,7 +85,7 @@ editCabalFilesFeature _env templates ok $ toResponse $ template [ "pkgid" $= pkgid , "cabalfile" $= insertRevisionField (pkgNumRevisions pkg) - (cabalFileByteString (pkgLatestCabalFileText pkg)) + (BS.L.fromStrict (cabalFileByteString (pkgLatestCabalFileText pkg))) ] serveEditCabalFilePost :: DynamicPath -> ServerPartE Response @@ -98,11 +99,11 @@ editCabalFilesFeature _env templates uid <- guardAuthorised [ InGroup (maintainersGroup pkgname) , InGroup trusteesGroup ] let oldVersion = cabalFileByteString (pkgLatestCabalFileText pkg) - newRevision <- getCabalFile + newRevision <- BS.L.toStrict <$> getCabalFile shouldPublish <- getPublish case diffCabalRevisionsByteString oldVersion newRevision of Left errs -> - responseTemplate template pkgid newRevision + responseTemplate template pkgid (BS.L.fromStrict newRevision) shouldPublish [errs] [] Right changes @@ -117,7 +118,7 @@ editCabalFilesFeature _env templates , "changes" $= changes ] | otherwise -> - responseTemplate template pkgid newRevision + responseTemplate template pkgid (BS.L.fromStrict newRevision) shouldPublish [] changes where @@ -126,7 +127,7 @@ editCabalFilesFeature _env templates (look "publish" >> return True) responseTemplate :: ([TemplateAttr] -> Template) -> PackageId - -> ByteString -> Bool -> [String] -> [Change] + -> LazyByteString -> Bool -> [String] -> [Change] -> ServerPartE Response responseTemplate template pkgid cabalFile publish errors changes = ok $ toResponse $ template @@ -139,11 +140,11 @@ editCabalFilesFeature _env templates -- | Wrapper around 'diffCabalRevisions' which operates on --- 'ByteString' decoded with lenient UTF8 and with any leading BOM +-- 'LazyByteString' decoded with lenient UTF8 and with any leading BOM -- stripped. -diffCabalRevisionsByteString :: ByteString -> ByteString -> Either String [Change] +diffCabalRevisionsByteString :: StrictByteString -> StrictByteString -> Either String [Change] diffCabalRevisionsByteString oldRevision newRevision = - maybe (diffCabalRevisions (BS.L.toStrict oldRevision) (BS.L.toStrict newRevision)) + maybe (diffCabalRevisions oldRevision newRevision) Left parseSpecVerCheck where diff --git a/src/Distribution/Server/Features/Html.hs b/src/Distribution/Server/Features/Html.hs index 8a187cbdd..84cdb9882 100644 --- a/src/Distribution/Server/Features/Html.hs +++ b/src/Distribution/Server/Features/Html.hs @@ -68,7 +68,7 @@ import qualified Data.Map as Map import qualified Data.Set as Set import qualified Data.Vector as Vec import qualified Data.Text as T -import qualified Data.ByteString.Lazy.Char8 as BS (ByteString) +import qualified Data.ByteString.Lazy as BS (LazyByteString, fromStrict) import qualified Network.URI as URI import Text.XHtml.Strict @@ -812,9 +812,9 @@ mkHtmlCore ServerEnv{serverBaseURI, serverBlobStore} start [] = [] start (curr:rest) = go curr rest - go curr [] = [(sha256 (cabalFileByteString (fst curr)), [])] + go curr [] = [(sha256 (BS.fromStrict (cabalFileByteString (fst curr))), [])] go curr (prev:rest) = - ( sha256 (cabalFileByteString (fst curr)) + ( sha256 (BS.fromStrict (cabalFileByteString (fst curr))) , changes curr prev ) : go prev rest @@ -849,7 +849,7 @@ mkHtmlCore ServerEnv{serverBaseURI, serverBlobStore} -- | Common helper used by 'serveCandidatePage' and 'servePackagePage' -makeReadme :: MonadIO m => PackageRender -> m (Maybe BS.ByteString) +makeReadme :: MonadIO m => PackageRender -> m (Maybe BS.LazyByteString) makeReadme render = case rendReadme render of Just (tarfile, _, offset, _) -> either (\_err -> return Nothing) (return . Just . snd) =<< diff --git a/src/Distribution/Server/Features/Mirror.hs b/src/Distribution/Server/Features/Mirror.hs index 1f49c3c7d..0db7660cd 100644 --- a/src/Distribution/Server/Features/Mirror.hs +++ b/src/Distribution/Server/Features/Mirror.hs @@ -243,12 +243,12 @@ mirrorFeature ServerEnv{serverBlobStore = store} cabalPut dpath = do uid <- guardMirrorGroup pkgid :: PackageId <- packageInPath dpath - fileContent <- expectTextPlain + fileContent <- BS.L.toStrict <$> expectTextPlain time <- liftIO getCurrentTime let uploadData = (time, uid) filename = display pkgid <.> "cabal" - case runParseResult $ parseGenericPackageDescription $ BS.L.toStrict fileContent of + case runParseResult $ parseGenericPackageDescription fileContent of (_, Left (_, err NE.:| _)) -> badRequest (toResponse $ showPError filename err) (_, Right pkg) | pkgid /= packageId pkg -> errBadRequest "Wrong package Id" diff --git a/src/Distribution/Server/Features/PackageCandidates.hs b/src/Distribution/Server/Features/PackageCandidates.hs index d0550ebbe..2539ccfd6 100644 --- a/src/Distribution/Server/Features/PackageCandidates.hs +++ b/src/Distribution/Server/Features/PackageCandidates.hs @@ -32,6 +32,7 @@ import Distribution.Server.Packages.PackageIndex (PackageIndex) import qualified Distribution.Server.Framework.ResponseContentTypes as Resource import Distribution.Server.Features.Security.Migration +import Distribution.Server.Util.Parse (unpackUTF8) import Distribution.Server.Util.ServeTarball import Distribution.Server.Util.Markdown (renderMarkdown, supposedToBeMarkdown) import Distribution.Server.Pages.Template (hackagePage) @@ -40,10 +41,8 @@ import Distribution.Text import Distribution.Package import Distribution.Version +import qualified Data.ByteString.Lazy as BS (toStrict, fromStrict) import qualified Data.Text as T -import qualified Data.Text.Encoding as T -import qualified Data.Text.Encoding.Error as T -import qualified Data.ByteString.Lazy as BS (ByteString, toStrict) import qualified Text.XHtml.Strict as XHtml import Text.XHtml.Strict ((<<), (!)) import Data.Aeson (Value (..), object, toJSON, (.=)) @@ -383,7 +382,7 @@ candidatesFeature ServerEnv{serverBlobStore = store} pkg <- packageInPath dpath >>= lookupCandidateId guard (lookup "cabal" dpath == Just (display $ packageName pkg)) let (fileRev, (utime, _uid)) = pkgLatestRevision (candPkgInfo pkg) - cabalfile = Resource.CabalFile (cabalFileByteString fileRev) utime + cabalfile = Resource.CabalFile (BS.fromStrict (cabalFileByteString fileRev)) utime return $ toResponse cabalfile uploadCandidate :: (PackageId -> Bool) -> ServerPartE CandPkgInfo @@ -396,7 +395,7 @@ candidatesFeature ServerEnv{serverBlobStore = store} now <- liftIO getCurrentTime let (UploadResult pkg pkgStr _) = uresult pkgid = packageId pkg - cabalfile = CabalFileText pkgStr + cabalfile = CabalFileText $ BS.toStrict pkgStr uploadinfo = (now, uid) candidate = CandPkgInfo { candPkgInfo = PkgInfo { @@ -453,7 +452,7 @@ candidatesFeature ServerEnv{serverBlobStore = store} -- run filters let pkgInfo = candPkgInfo candidate uresult = UploadResult (pkgDesc pkgInfo) - (cabalFileByteString (pkgLatestCabalFileText pkgInfo)) + (BS.fromStrict (cabalFileByteString (pkgLatestCabalFileText pkgInfo))) (candWarnings candidate) time <- liftIO getCurrentTime let uploadInfo = (time, uid) @@ -596,7 +595,7 @@ candidatesFeature ServerEnv{serverBlobStore = store} << if supposedToBeMarkdown filename then renderMarkdown filename contents else XHtml.thediv ! [XHtml.theclass "preformatted"] - << unpackUtf8 contents + << unpackUTF8 contents ] @@ -614,8 +613,3 @@ candidatesFeature ServerEnv{serverBlobStore = store} ["index.html"] (display (packageId pkg)) fp index [Public, maxAgeMinutes 5] etag Nothing requireUserContent userFeatureServerEnv (tarServeResponse tarServe) - -unpackUtf8 :: BS.ByteString -> String -unpackUtf8 = T.unpack - . T.decodeUtf8With T.lenientDecode - . BS.toStrict diff --git a/src/Distribution/Server/Features/PackageContents.hs b/src/Distribution/Server/Features/PackageContents.hs index 392afb2a9..43b2df134 100644 --- a/src/Distribution/Server/Features/PackageContents.hs +++ b/src/Distribution/Server/Features/PackageContents.hs @@ -18,16 +18,13 @@ import Distribution.Server.Packages.Render import Distribution.Server.Features.Users import Distribution.Server.Util.ServeTarball import Distribution.Server.Util.Markdown (renderMarkdown, supposedToBeMarkdown) +import Distribution.Server.Util.Parse (unpackUTF8) import Distribution.Server.Pages.Template (hackagePage) import Distribution.Text import Distribution.Package import Distribution.PackageDescription -import qualified Data.Text as T -import qualified Data.Text.Encoding as T -import qualified Data.Text.Encoding.Error as T -import qualified Data.ByteString.Lazy as BS (ByteString, toStrict) import qualified Text.XHtml.Strict as XHtml import qualified Distribution.Utils.ShortText as ST import Text.XHtml.Strict ((<<), (!)) @@ -160,7 +157,7 @@ packageContentsFeature CoreFeature{ coreResource = CoreResource{ << if supposedToBeMarkdown filename then renderMarkdown filename contents else XHtml.thediv ! [XHtml.theclass "preformatted"] - << unpackUtf8 contents + << unpackUTF8 contents ] serveReadmeText :: DynamicPath -> ServerPartE Response @@ -194,7 +191,7 @@ packageContentsFeature CoreFeature{ coreResource = CoreResource{ << if supposedToBeMarkdown filename then renderMarkdown filename contents else XHtml.thediv ! [XHtml.theclass "preformatted"] - << unpackUtf8 contents + << unpackUTF8 contents ] -- return: not-found error or tarball @@ -212,11 +209,6 @@ packageContentsFeature CoreFeature{ coreResource = CoreResource{ [Public, maxAgeDays 30] etag Nothing requireUserContent userFeatureServerEnv (tarServeResponse tarServe) -unpackUtf8 :: BS.ByteString -> String -unpackUtf8 = T.unpack - . T.decodeUtf8With T.lenientDecode - . BS.toStrict - -- TODO: this helper is defined in at least two other places; consolidate -- | URL describing a package. packageURL :: PackageIdentifier -> XHtml.URL diff --git a/src/Distribution/Server/Features/PackageFeed.hs b/src/Distribution/Server/Features/PackageFeed.hs index a71cf5f06..d99432629 100644 --- a/src/Distribution/Server/Features/PackageFeed.hs +++ b/src/Distribution/Server/Features/PackageFeed.hs @@ -10,6 +10,7 @@ import Distribution.Server.Packages.ChangeLog import Distribution.Server.Packages.Types import qualified Distribution.Server.Users.Users as Users import Distribution.Server.Users.Users (Users) +import Distribution.Server.Util.Parse (unpackUTF8) import Distribution.Server.Util.ServeTarball (loadTarEntry) import Distribution.Server.Util.Markdown (renderMarkdown, supposedToBeMarkdown) import Distribution.Server.Pages.Package () -- for ShortText html instance, for now. @@ -19,13 +20,9 @@ import Distribution.PackageDescription import Distribution.Text import Distribution.Utils.ShortText (fromShortText) -import qualified Data.ByteString.Lazy as BS (ByteString, toStrict) import Data.List (sortOn) import Data.Maybe (listToMaybe) import Data.Ord (Down(..)) -import qualified Data.Text as T -import qualified Data.Text.Encoding as T -import qualified Data.Text.Encoding.Error as T import Data.Time.Clock (UTCTime, getCurrentTime) import Data.Time.Format import Network.URI( URI(..), uriToString ) @@ -96,7 +93,7 @@ packageFeedFeature ServerEnv{..} Right (_, content) -> if supposedToBeMarkdown filename then return (pkg, renderMarkdown filename content) - else return (pkg, XHtml.pre << unpackUtf8 content) + else return (pkg, XHtml.pre << unpackUTF8 content) renderPackageFeed :: Users -> URI -> UTCTime -> PackageName -> [(PkgInfo, XHtml.Html)] -> RSS renderPackageFeed users hostURI now name pkgs = RSS title uri desc (channel updated) items @@ -139,9 +136,3 @@ feedItems users hostURI (pkgInfo, chlog) = uploader = display $ Users.userIdToName users uploaderId pd = packageDescription (pkgDesc pkgInfo) d dt dd = XHtml.dterm (XHtml.toHtml dt) +++ XHtml.ddef (XHtml.toHtml dd) - - -unpackUtf8 :: BS.ByteString -> String -unpackUtf8 = T.unpack - . T.decodeUtf8With T.lenientDecode - . BS.toStrict diff --git a/src/Distribution/Server/Features/PackageInfoJSON.hs b/src/Distribution/Server/Features/PackageInfoJSON.hs index 3e3b508f2..58607529d 100644 --- a/src/Distribution/Server/Features/PackageInfoJSON.hs +++ b/src/Distribution/Server/Features/PackageInfoJSON.hs @@ -19,7 +19,6 @@ import Distribution.Server.Prelude import qualified Data.Aeson as Aeson import Data.Aeson ((.=)) import qualified Data.Aeson.Key as Key -import qualified Data.ByteString.Lazy.Char8 as BS (toStrict) import qualified Data.Map.Strict as Map import qualified Data.Text as T import qualified Data.Vector as Vector @@ -181,7 +180,7 @@ getBasicDescription -- @PackageBasicDescription@ -> Either String PackageBasicDescription getBasicDescription uploadedAt (CabalFileText cf) metadataRev = - let parseResult = PkgDescr.parseGenericPackageDescription (BS.toStrict cf) + let parseResult = PkgDescr.parseGenericPackageDescription cf in case PkgDescr.runParseResult parseResult of (_, Right pkg) -> let pkgd = PkgDescr.packageDescription pkg diff --git a/src/Distribution/Server/Features/Upload.hs b/src/Distribution/Server/Features/Upload.hs index 7bb1f7fe8..d13d1fad8 100644 --- a/src/Distribution/Server/Features/Upload.hs +++ b/src/Distribution/Server/Features/Upload.hs @@ -29,7 +29,7 @@ import Data.Maybe (fromMaybe) import Data.List (dropWhileEnd, intersperse) import Data.Time.Clock (getCurrentTime) import Data.Function (fix) -import Data.ByteString.Lazy (ByteString) +import Data.ByteString.Lazy (LazyByteString, toStrict) import Distribution.Package import Distribution.PackageDescription (GenericPackageDescription) @@ -97,7 +97,7 @@ data UploadResult = UploadResult { -- The parsed Cabal file. uploadDesc :: !GenericPackageDescription, -- The text of the Cabal file. - uploadCabal :: !ByteString, + uploadCabal :: !LazyByteString, -- Any warnings from unpacking the tarball. uploadWarnings :: ![String] } @@ -302,7 +302,7 @@ uploadFeature ServerEnv{serverBlobStore = store} now <- liftIO getCurrentTime let (UploadResult pkg pkgStr _) = uresult pkgid = packageId pkg - cabalfile = CabalFileText pkgStr + cabalfile = CabalFileText $ toStrict pkgStr uploadinfo = (now, uid) success <- updateAddPackage pkgid cabalfile uploadinfo (Just tarball) if success @@ -417,7 +417,7 @@ uploadFeature ServerEnv{serverBlobStore = store} --FIXME: this should have been covered earlier uid <- guardAuthenticated now <- liftIO getCurrentTime - let processPackage :: ByteString -> IO (Either ErrorResponse (UploadResult, BlobStorage.BlobId)) + let processPackage :: LazyByteString -> IO (Either ErrorResponse (UploadResult, BlobStorage.BlobId)) processPackage content' = do -- as much as it would be nice to do requirePackageAuth in here, -- processPackage is run in a handle bracket diff --git a/src/Distribution/Server/Packages/Index.hs b/src/Distribution/Server/Packages/Index.hs index a822b3250..a1e143f94 100644 --- a/src/Distribution/Server/Packages/Index.hs +++ b/src/Distribution/Server/Packages/Index.hs @@ -24,15 +24,12 @@ import Distribution.Server.Users.Users ( Users, userIdToName ) import Distribution.Server.Users.Types ( UserId(..), UserName(..) ) -import Distribution.Server.Util.ParseSpecVer import Distribution.Text ( display ) import Distribution.Types.PackageName import Distribution.Package ( Package, PackageId, packageName, packageVersion ) -import Distribution.CabalSpecVersion - ( pattern CabalSpecV2_0 ) import Data.Time.Clock ( UTCTime ) import Data.Time.Clock.POSIX @@ -43,7 +40,7 @@ import Data.SafeCopy (base, deriveSafeCopy) import Data.Map (Map) import qualified Data.Map as Map import qualified Data.Vector as Vec -import Data.ByteString.Lazy (ByteString) +import Data.ByteString.Lazy (LazyByteString, fromStrict) import System.FilePath.Posix import Data.Maybe (mapMaybe) @@ -76,7 +73,7 @@ data TarIndexEntry = -- | Additional entries that we add to the tarball -- -- This is currently used for @preferred-versions@. - | ExtraEntry !FilePath !ByteString !UTCTime + | ExtraEntry !FilePath !LazyByteString !UTCTime deriving (Eq, Show) type RevisionNo = Int @@ -92,7 +89,7 @@ deriveSafeCopy 0 'base ''TarIndexEntry -- a package index, an index tarball. This tarball has the modification times -- and uploading users built-in. -writeIncremental :: PackageIndex PkgInfo -> [TarIndexEntry] -> ByteString +writeIncremental :: PackageIndex PkgInfo -> [TarIndexEntry] -> LazyByteString writeIncremental pkgs = Tar.write . mapMaybe mkTarEntry where @@ -107,7 +104,7 @@ writeIncremental pkgs = tarPath <- either (const Nothing) Just $ Tar.toTarPath False fileName let !tarEntry = addTimestampAndOwner timestamp userid username $ - Tar.fileEntry tarPath cabalfile + Tar.fileEntry tarPath $ fromStrict cabalfile return tarEntry where pkgname = unPackageName (packageName pkgid) @@ -144,14 +141,14 @@ utcToUnixTime :: UTCTime -> Int64 utcToUnixTime = truncate . utcTimeToPOSIXSeconds -- | Extract legacy entries -legacyExtras :: [TarIndexEntry] -> Map String (ByteString, UTCTime) +legacyExtras :: [TarIndexEntry] -> Map String (LazyByteString, UTCTime) legacyExtras = go Map.empty where -- Later entries in the update log will override earlier ones. This is -- intentional. - go :: Map String (ByteString, UTCTime) + go :: Map String (LazyByteString, UTCTime) -> [TarIndexEntry] - -> Map String (ByteString, UTCTime) + -> Map String (LazyByteString, UTCTime) go acc [] = acc go acc (ExtraEntry fp bs time : es) = let acc' = Map.insert fp (bs, time) acc @@ -173,9 +170,9 @@ legacyExtras = go Map.empty -- compression), contains at most one preferred-version per package (important -- because of a bug in cabal which would otherwise merge all preferred-versions -- files for a package), and does not contain the TUF files. -writeLegacy :: Users -> Map String (ByteString, UTCTime) -> PackageIndex PkgInfo -> ByteString +writeLegacy :: Users -> Map String (LazyByteString, UTCTime) -> PackageIndex PkgInfo -> LazyByteString writeLegacy users = - writeLegacyAux (cabalFileByteString . pkgLatestCabalFileText) setModTime + writeLegacyAux (fromStrict . cabalFileByteString . pkgLatestCabalFileText) setModTime . extraEntries where setModTime pkgInfo entry = @@ -192,13 +189,13 @@ writeLegacy users = userName = display . userIdToName users - extraEntries :: Map FilePath (ByteString, UTCTime) -> [Tar.Entry] + extraEntries :: Map FilePath (LazyByteString, UTCTime) -> [Tar.Entry] extraEntries emap = do (path, (entry, mtime)) <- Map.toList emap Right tarPath <- return $ Tar.toTarPath False path return $ (Tar.fileEntry tarPath entry) { Tar.entryTime = utcToUnixTime mtime } --- | Create an uncompressed tar repository index file as a 'ByteString'. +-- | Create an uncompressed tar repository index file as a 'LazyByteString'. -- -- Takes a couple functions to turn a package into a tar entry. Extra -- entries are also accepted. @@ -206,11 +203,11 @@ writeLegacy users = -- This used to live in Distribution.Server.Util.Index. -- writeLegacyAux :: Package pkg - => (pkg -> ByteString) + => (pkg -> LazyByteString) -> (pkg -> Tar.Entry -> Tar.Entry) -> [Tar.Entry] -> PackageIndex pkg - -> ByteString + -> LazyByteString writeLegacyAux externalPackageRep updateEntry extras = Tar.write . (extras++) . map entry . PackageIndex.allPackages where diff --git a/src/Distribution/Server/Packages/Types.hs b/src/Distribution/Server/Packages/Types.hs index 1bf06236e..32cdde48c 100644 --- a/src/Distribution/Server/Packages/Types.hs +++ b/src/Distribution/Server/Packages/Types.hs @@ -21,7 +21,7 @@ import Distribution.Server.Users.Types (UserId(..)) import Distribution.Server.Framework.BlobStorage (BlobId, BlobId_v0, BlobStorage) import Distribution.Server.Framework.Instances (PackageIdentifier_v0) import Distribution.Server.Framework.MemSize -import Distribution.Server.Util.Parse (unpackUTF8) +import Distribution.Server.Util.Parse (unpackUTF8Strict) import Distribution.Server.Features.Security.Orphans () import Distribution.Server.Features.Security.MD5 import Distribution.Server.Features.Security.SHA256 @@ -35,7 +35,8 @@ import Distribution.PackageDescription.Parsec ( parseGenericPackageDescription, runParseResult ) import Data.Serialize (Serialize) -import Data.ByteString.Lazy (ByteString) +import Data.ByteString (StrictByteString) +import Data.ByteString.Lazy (LazyByteString) import Data.Time.Clock (UTCTime(..)) import Data.Time.Calendar (Day(..)) import Data.SafeCopy @@ -47,7 +48,9 @@ import qualified Data.Vector as Vec Datatypes -------------------------------------------------------------------------------} -newtype CabalFileText = CabalFileText { cabalFileByteString :: ByteString } +-- | Cabal files are definitely small enough to use a strict ByteString. +-- This eliminates one possible source of issues with lazy IO. +newtype CabalFileText = CabalFileText { cabalFileByteString :: StrictByteString } deriving (Eq, MemSize) -- | The information we keep about a particular version of a package. @@ -156,7 +159,7 @@ instance Package PkgInfo where -------------------------------------------------------------------------------} cabalFileString :: CabalFileText -> String -cabalFileString = unpackUTF8 . cabalFileByteString +cabalFileString = unpackUTF8Strict . cabalFileByteString pkgOriginalRevision :: PkgInfo -> (CabalFileText, UploadInfo) pkgOriginalRevision = Vec.head . pkgMetadataRevisions @@ -208,7 +211,7 @@ pkgLatestTarball pkginfo = pkgDesc :: PkgInfo -> GenericPackageDescription pkgDesc pkgInfo = case runParseResult $ parseGenericPackageDescription $ - BS.L.toStrict $ cabalFileByteString $ fst $ + cabalFileByteString $ fst $ pkgLatestRevision pkgInfo of -- We only make PkgInfos with parsable pkgDatas, so if it -- doesn't parse then something has gone wrong. @@ -219,7 +222,7 @@ pkgDesc pkgInfo = pkgDescMaybe :: PkgInfo -> Maybe GenericPackageDescription pkgDescMaybe pkgInfo = case runParseResult $ parseGenericPackageDescription $ - BS.L.toStrict $ cabalFileByteString $ fst $ + cabalFileByteString $ fst $ pkgLatestRevision pkgInfo of -- We only make PkgInfos with parsable pkgDatas, so if it -- doesn't parse then something has gone wrong. @@ -227,7 +230,7 @@ pkgDescMaybe pkgInfo = (_, Right x) -> Just x -blobInfoFromBS :: BlobId -> ByteString -> BlobInfo +blobInfoFromBS :: BlobId -> LazyByteString -> BlobInfo blobInfoFromBS blobId bs = BlobInfo { blobInfoId = blobId , blobInfoLength = fromIntegral $ BS.L.length bs diff --git a/src/Distribution/Server/Packages/Unpack.hs b/src/Distribution/Server/Packages/Unpack.hs index 029393b56..750af4894 100644 --- a/src/Distribution/Server/Packages/Unpack.hs +++ b/src/Distribution/Server/Packages/Unpack.hs @@ -183,7 +183,7 @@ basicChecks pkgid tarIndex = do ++ "save the package's cabal file as UTF8 without the BOM." -- Parse the Cabal file - (specVerOk,pkgDesc, warnings) <- case parseGenericPackageDescriptionChecked cabalEntry of + (specVerOk,pkgDesc, warnings) <- case parseGenericPackageDescriptionChecked (LBS.toStrict cabalEntry) of (_, _, Left (_, err:_)) -> -- TODO: show all errors throwError $ showPError cabalFileName err (_, _, Left (_, [])) -> diff --git a/src/Distribution/Server/Pages/PackageFromTemplate.hs b/src/Distribution/Server/Pages/PackageFromTemplate.hs index 82ac34744..edaa7a07b 100644 --- a/src/Distribution/Server/Pages/PackageFromTemplate.hs +++ b/src/Distribution/Server/Pages/PackageFromTemplate.hs @@ -20,6 +20,7 @@ import Distribution.Server.Packages.Types import Distribution.Server.Features.PackageCandidates import Distribution.Server.Users.Types (UserInfo, userStatus, userName, isActiveAccount) import Distribution.Server.Util.Markdown (renderMarkdown, supposedToBeMarkdown) +import Distribution.Server.Util.Parse (unpackUTF8) import Data.TarIndex (TarIndex) import Distribution.Server.Features.Distro.Types @@ -37,10 +38,7 @@ import Data.List (intercalate, intersperse) import System.FilePath.Posix ((), takeFileName, dropTrailingPathSeparator) import Data.Time.Format (defaultTimeLocale, formatTime) -import qualified Data.Text as T -import qualified Data.Text.Encoding as T -import qualified Data.Text.Encoding.Error as T -import qualified Data.ByteString.Lazy as BS (ByteString, toStrict) +import qualified Data.ByteString.Lazy as BS (ByteString) import qualified Distribution.Server.Pages.Package as Old import Data.Time.Clock (UTCTime) @@ -500,15 +498,10 @@ readmeSection PackageRender { rendReadme = Just (_, _etag, _, filename), rendPkg [ thediv ! [theclass "embedded-author-content"] << if supposedToBeMarkdown filename then renderMarkdown (display pkgid) content - else pre << unpackUtf8 content + else pre << unpackUTF8 content ] readmeSection _ _ = [] - -unpackUtf8 :: BS.ByteString -> String -unpackUtf8 = T.unpack - . T.decodeUtf8With T.lenientDecode - . BS.toStrict ----------------------------------------------------------------------------- commaList :: [Html] -> Html commaList = concatHtml . intersperse (toHtml ", ") diff --git a/src/Distribution/Server/Util/Parse.hs b/src/Distribution/Server/Util/Parse.hs index 83b04a4e4..80297f6ef 100644 --- a/src/Distribution/Server/Util/Parse.hs +++ b/src/Distribution/Server/Util/Parse.hs @@ -1,14 +1,17 @@ -- | Parsing and UTF8 utilities module Distribution.Server.Util.Parse ( - int, unpackUTF8, packUTF8 + int, unpackUTF8, unpackUTF8Strict, packUTF8 ) where import qualified Text.ParserCombinators.ReadP as Parse import qualified Data.Char as Char -import Data.ByteString.Lazy (ByteString) -import qualified Data.Text.Lazy as Text -import qualified Data.Text.Lazy.Encoding as Text +import Data.ByteString (StrictByteString) +import Data.ByteString.Lazy (LazyByteString) +import qualified Data.Text as TextStrict +import qualified Data.Text.Encoding as TextStrict +import qualified Data.Text.Lazy as TextLazy +import qualified Data.Text.Lazy.Encoding as TextLazy import qualified Data.Text.Encoding.Error as Text -- | Parse a positive integer. No leading @0@'s allowed. @@ -28,8 +31,11 @@ ignoreBOM :: String -> String ignoreBOM ('\xFEFF':string) = string ignoreBOM string = string -unpackUTF8 :: ByteString -> String -unpackUTF8 = ignoreBOM . Text.unpack . Text.decodeUtf8With Text.lenientDecode +unpackUTF8 :: LazyByteString -> String +unpackUTF8 = ignoreBOM . TextLazy.unpack . TextLazy.decodeUtf8With Text.lenientDecode -packUTF8 :: String -> ByteString -packUTF8 = Text.encodeUtf8 . Text.pack +unpackUTF8Strict :: StrictByteString -> String +unpackUTF8Strict = ignoreBOM . TextStrict.unpack . TextStrict.decodeUtf8With Text.lenientDecode + +packUTF8 :: String -> LazyByteString +packUTF8 = TextLazy.encodeUtf8 . TextLazy.pack diff --git a/src/Distribution/Server/Util/ParseSpecVer.hs b/src/Distribution/Server/Util/ParseSpecVer.hs index bf6ca9f4e..2aff7e5e9 100644 --- a/src/Distribution/Server/Util/ParseSpecVer.hs +++ b/src/Distribution/Server/Util/ParseSpecVer.hs @@ -6,17 +6,15 @@ module Distribution.Server.Util.ParseSpecVer ( parseSpecVer , parseSpecVerLazy , scanSpecVersion - , scanSpecVersionLazy , parseGenericPackageDescriptionChecked ) where import Distribution.Server.Prelude -import Data.ByteString (ByteString) +import Data.ByteString (StrictByteString) import qualified Data.ByteString as BS import qualified Data.ByteString.Char8 as BC8 import qualified Data.ByteString.Lazy as BSL -import qualified Data.ByteString.Lazy.Char8 as BC8L import Data.List.NonEmpty ( toList) import Distribution.Text import Distribution.CabalSpecVersion @@ -47,10 +45,10 @@ import Distribution.PackageDescription.Parsec ( runParseResult, parseG -- @.cabal@ files are accepted which support the heuristic parsing. -- -- If no valid version field can be found, @Nothing@ is returned. -parseSpecVer :: ByteString -> Maybe CabalSpecVersion +parseSpecVer :: StrictByteString -> Maybe CabalSpecVersion parseSpecVer = findCabVer -parseSpecVerLazy :: BSL.ByteString -> Maybe CabalSpecVersion +parseSpecVerLazy :: BSL.LazyByteString -> Maybe CabalSpecVersion parseSpecVerLazy = parseSpecVer . BSL.toStrict versionToCabalSpecVersion :: Version -> Maybe CabalSpecVersion @@ -59,11 +57,11 @@ versionToCabalSpecVersion = cabalSpecFromVersionDigits . versionNumbers isWS :: Word8 -> Bool isWS = (`elem` [0x20,0x09]) -eatWS :: ByteString -> ByteString +eatWS :: StrictByteString -> StrictByteString eatWS = BS.dropWhile isWS -- | Try to heuristically locate & parse a 'cabal-version' field -findCabVer :: ByteString -> Maybe CabalSpecVersion +findCabVer :: StrictByteString -> Maybe CabalSpecVersion findCabVer raw = msum [ versionToCabalSpecVersion =<< decodeVer y | (_,_,y) <- findCabVers raw ] -- | Return list of @cabal-version@ candidates as 3-tuples of @@ -81,7 +79,7 @@ findCabVer raw = msum [ versionToCabalSpecVersion =<< decodeVer y | (_,_,y) <- f -- -- NB: Later occurrences of @cabal-version@ override earlier ones. In -- future @cabal-versions@ it will be disallowed. -findCabVers :: ByteString -> [(ByteString,Int,[ByteString])] +findCabVers :: StrictByteString -> [(StrictByteString, Int, [StrictByteString])] findCabVers buf0 = mapMaybe go ixs where go i @@ -105,7 +103,7 @@ findCabVers buf0 = mapMaybe go ixs map getInd l' -- split off indentation for single line - getInd :: ByteString -> (Int,ByteString) + getInd :: StrictByteString -> (Int, StrictByteString) getInd x = case BS.span isWS x of (i,r) -> (BS.length i,r) isNonComment = not . BS.isPrefixOf "--" @@ -114,11 +112,11 @@ findCabVers buf0 = mapMaybe go ixs ixs = strCaseStrAll buf0 "cabal-version" -- | Lookup-table mapping "x.y.z" strings to 'Version' -verDictV :: Map.HashMap ByteString Version +verDictV :: Map.HashMap StrictByteString Version verDictV = Map.fromList [ (BC8.pack (prettyShow v), v) | v <- knownVers ] -- | Lookup-table mapping ">=x.y.z" strings to 'Version' -verDictRg :: Map.HashMap ByteString Version +verDictRg :: Map.HashMap StrictByteString Version verDictRg = Map.fromList [ (">=" <> BC8.pack (prettyShow v), v) | v <- knownVers ] -- | List of cabal-version values contained in Hackage's package index as of 2017-07 @@ -224,7 +222,7 @@ knownVers = map mkVersion ] -- | Fast decoder -decodeVer :: [ByteString] -> Maybe Version +decodeVer :: [StrictByteString] -> Maybe Version decodeVer ws = case ws of [">=",v] -> Map.lookup v verDictV -- most common case [v] -> Map.lookup v verDictRg <|> -- most common case @@ -233,7 +231,7 @@ decodeVer ws = case ws of _ -> decodeVerFallback (mconcat ws) -- | Fallback parser for when lookup-table based parsing fails -decodeVerFallback :: ByteString -> Maybe Version +decodeVerFallback :: StrictByteString -> Maybe Version decodeVerFallback v0 = simpleParse v <|> parseSpecVR where parseSpecVR = do @@ -250,7 +248,7 @@ foreign import ccall unsafe "string.h strcasestr" c_strcasestr :: Ptr CChar -> P -- | Find indices (in reverse order) of all non-overlapping -- case-insensitive occurrences of s2 in s1 {-# NOINLINE strCaseStrAll #-} -strCaseStrAll :: ByteString -> ByteString -> [Int] +strCaseStrAll :: StrictByteString -> StrictByteString -> [Int] strCaseStrAll s1 s2 | BS.null s1 || BS.null s2 = [] | BS.elem 0 s1 || BS.elem 0 s2 = undefined @@ -271,35 +269,27 @@ strCaseStrAll s1 s2 let !ofs = m `minusPtr` hay0 go (m `plusPtr` needleSz) (ofs:acc) - --- | Lazy 'BSL.ByteString' version of 'scanSpecVersion' -scanSpecVersionLazy :: BSL.ByteString -> Maybe Version -scanSpecVersionLazy bs = do - fstline':_ <- pure (BC8L.lines bs) - scanSpecVersion (BSL.toStrict fstline') - - -- | Version of 'parseGenericPackageDescription' which also validates spec-version heuristics -- -- * Result of 'parseSpecVerLazy' must agree with 'parseGenericPackageDescription' --- * If 'scanSpecVersionLazy' detects a version, then it must agree with 'parseGenericPackageDescription' as well --- * Starting with cabal-version:2.2 'scanSpecVersionLazy' must succeed +-- * If 'scanSpecVersion' detects a version, then it must agree with 'parseGenericPackageDescription' as well +-- * Starting with cabal-version:2.2 'scanSpecVersion' must succeed -- -- 'True' is returned in the first element if sanity checks passes. -parseGenericPackageDescriptionChecked :: BSL.ByteString -> (Bool, [PWarning], Either (Maybe Version, [PError]) GenericPackageDescription) +parseGenericPackageDescriptionChecked :: StrictByteString -> (Bool, [PWarning], Either (Maybe Version, [PError]) GenericPackageDescription) parseGenericPackageDescriptionChecked bs = case parseGenericPackageDescription' bs of (warns, Left pe) -> (False, warns, Left $ fmap toList pe) (warns, Right gpd) -> (isOk (specVersion (packageDescription gpd)), warns, Right gpd) where isOk :: CabalSpecVersion -> Bool isOk v - | Just v /= parseSpecVerLazy bs = False - | Just v' <- versionToCabalSpecVersion =<< scanSpecVersionLazy bs + | Just v /= parseSpecVer bs = False + | Just v' <- versionToCabalSpecVersion =<< scanSpecVersion bs = v == v' | otherwise = v <= CabalSpecV2_2 #if defined(MIN_VERSION_cabal_parsers) - parseGenericPackageDescription' bs' = compatParseGenericPackageDescription (BSL.toStrict bs') + parseGenericPackageDescription' bs' = compatParseGenericPackageDescription bs' #else - parseGenericPackageDescription' bs' = runParseResult (parseGenericPackageDescription (BSL.toStrict bs')) + parseGenericPackageDescription' bs' = runParseResult (parseGenericPackageDescription bs') #endif diff --git a/tests/RevDepCommon.hs b/tests/RevDepCommon.hs index 9a9ac3330..c4730717a 100644 --- a/tests/RevDepCommon.hs +++ b/tests/RevDepCommon.hs @@ -48,7 +48,7 @@ mkPackageWithCabalFileSuffix name intVersion cabalFileSuffix = \name: " <> BSL.fromStrict (Char8.pack $ unPackageName name) <> "\n\ \version: " <> dotVersion <> "\n" cabalFile :: CabalFileText - cabalFile = CabalFileText $ cabalFilePrefix <> cabalFileSuffix + cabalFile = CabalFileText $ BSL.toStrict $ cabalFilePrefix <> cabalFileSuffix in PkgInfo (PackageIdentifier name version) diff --git a/tests/ReverseDependenciesTest.hs b/tests/ReverseDependenciesTest.hs index fe78692e3..82f3100b8 100644 --- a/tests/ReverseDependenciesTest.hs +++ b/tests/ReverseDependenciesTest.hs @@ -9,7 +9,6 @@ import qualified Control.Monad.Trans.State as State import qualified Data.Array as Arr import qualified Data.Bimap as Bimap import qualified Data.ByteString.Lazy as Lazy (ByteString) -import qualified Data.ByteString.Lazy as ByteStringL import Data.Foldable (for_) import Data.Functor.Identity (Identity(..)) import Data.List (partition, foldl') @@ -551,7 +550,7 @@ getNotificationEmailsTests = genPackageName = mkPackageName <$> Gen.string (Range.linear 1 30) Gen.unicode genVersion = mkVersion <$> Gen.list (Range.linear 1 4) (Gen.int $ Range.linear 0 50) genPackageId = PackageIdentifier <$> genPackageName <*> genVersion - genCabalFileText = CabalFileText . ByteStringL.fromStrict <$> Gen.utf8 (Range.linear 0 50000) Gen.unicode + genCabalFileText = CabalFileText <$> Gen.utf8 (Range.linear 0 50000) Gen.unicode genNonExistentUserId = UserId <$> Gen.int (Range.linear (-1000) (-1)) genUploadInfo = (,) <$> genUTCTime <*> genNonExistentUserId genTag = Tag <$> Gen.string (Range.linear 1 10) Gen.unicode