Simplify packing

Author: zooba

windows-release/azure-pipelines.yml

@@ -183,12 +183,6 @@ stages:
           ${{ if eq(parameters.SignNuget, 'true') }}:
             SigningCertificate: ${{ parameters.SigningCertificate }}
         DoFreethreaded: ${{ parameters.DoFreethreaded }}
-    - template: stage-pack-pymanager.yml
-      parameters:
-        ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
-          SigningCertificate: ${{ parameters.SigningCertificate }}
-        DoFreethreaded: ${{ parameters.DoFreethreaded }}
-        DoEmbed: ${{ parameters.DoEmbed }}
 
   - stage: Test
     dependsOn: Pack

windows-release/merge-and-upload.py

@@ -151,7 +151,11 @@ def purge(url):
 
 
 def calculate_uploads():
-    for p in sorted(Path().absolute().glob("__install__.*.json")):
+    for p in sorted(Path().absolute().iterdir()):
+        p /= "__install__.json"
+        if not p.is_file():
+            continue
+        print("Processing", p)
         i = json.loads(p.read_bytes())
         u = urlparse(i["url"])
         src = Path(u.path.rpartition("/")[-1]).absolute()

windows-release/stage-layout-pymanager.yml

@@ -39,6 +39,26 @@ jobs:
         IncludeDoc: true
         ${{ if eq(parameters.ARM64TclTk, 'true') }}:
           TclLibrary: tcltk_lib_arm64\tcl8
+      win32_test:
+        Name: win32_test
+        Arch: win32
+        TclLibrary: tcltk_lib_win32\tcl8
+        LayoutOptions: '--preset-pymanager-test'
+        IncludeDoc: true
+      amd64_test:
+        Name: amd64_test
+        Arch: amd64
+        TclLibrary: tcltk_lib_amd64\tcl8
+        LayoutOptions: '--preset-pymanager-test'
+        IncludeDoc: true
+      arm64_test:
+        Name: arm64_test
+        Arch: arm64
+        HostArch: amd64
+        LayoutOptions: '--preset-pymanager-test'
+        IncludeDoc: true
+        ${{ if eq(parameters.ARM64TclTk, 'true') }}:
+          TclLibrary: tcltk_lib_arm64\tcl8
       ${{ if eq(parameters.DoFreethreaded, 'true') }}:
         win32_t:
           Name: win32_t
@@ -80,6 +100,9 @@ jobs:
 
   steps:
   - template: ./checkout.yml
+    parameters:
+      IncludeSelf: true
+      Path: $(Build.SourcesDirectory)\cpython
 
   - download: current
     artifact: bin_$(Name)
@@ -103,14 +126,44 @@ jobs:
   - powershell: |
       copy "$(Pipeline.Workspace)\bin_$(Name)\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force
     displayName: 'Copy signed files into sources'
+    workingDirectory: $(Build.SourcesDirectory)\cpython
     condition: and(succeeded(), variables['SigningCertificate'])
 
   - template: ./layout-command.yml
+    parameters:
+      Sources: $(Build.SourcesDirectory)\cpython
 
-  - powershell: |
-      $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" $(LayoutOptions)
+  - powershell: >
+      $(LayoutCmd)
+      --copy "$(Build.ArtifactStagingDirectory)\layout"
+      --zip "$(Build.ArtifactStagingDirectory)\zip\package.zip"
+      $(LayoutOptions)
     displayName: 'Generate PyManager layout'
 
+  - powershell: |
+      # ConvertFrom-Json can't handle empty keys, but we don't need them anyway.
+      # Replace with an underscore so it can load.
+      $install = (gc -raw "layout\__install__.json") -replace '"":', '"_":' | ConvertFrom-Json
+      # Bring a copy of the install data separate from the ZIP
+      copy "layout\__install__.json" "zip\__install__.json"
+      # Rename the ZIP to match the target filename
+      # (which we didn't know when we named it package.zip)
+      $filename = Split-Path -Leaf $install.url
+      move "zip\package.zip" "zip\$filename"
+    displayName: 'Prepare PyManager distribution files'
+    workingDirectory: $(Build.ArtifactStagingDirectory)
+
+  - powershell: >
+      & $(Python) "$(Build.SourcesDirectory)\release-tools\sbom.py"
+      "--cpython-source-dir=$(Build.SourcesDirectory)\cpython"
+      $(gci "zip\*.zip")
+    workingDirectory: $(Build.ArtifactStagingDirectory)
+    displayName: 'Create SBOMs for package'
+
   - publish: '$(Build.ArtifactStagingDirectory)\layout'
     artifact: layout_pymanager_$(Name)
     displayName: 'Publish Artifact: layout_pymanager_$(Name)'
+
+  - publish: '$(Build.ArtifactStagingDirectory)\zip'
+    artifact: pymanager_$(Name)
+    displayName: 'Publish Artifact: pymanager_$(Name)'

windows-release/stage-pack-pymanager.yml

@@ -1,12 +1,11 @@
 parameters:
   BuildToPublish: ''
   HashAlgorithms: ['SHA256']
-  FeedPath: 'srv/ftp/download/__index_windows__.json'
 
 jobs:
-- job: Publish_Python
-  displayName: Publish python.org packages
-  condition: and(succeeded(), eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true'), ne(variables['SkipPythonOrgPublish'], 'true'))
+- job: Publish_PyManager
+  displayName: Publish PyManager packages to python.org
+  condition: and(succeeded(), ne(variables['SkipPythonOrgPublish'], 'true'))
 
   pool:
     vmImage: windows-2022
@@ -24,29 +23,23 @@ jobs:
       versionSpec: '>=3.10'
 
   - ${{ if parameters.BuildToPublish }}:
-    - task: DownloadPipelineArtifact@1
-      displayName: 'Download artifact from ${{ parameters.BuildToPublish }}: pymanager'
-      inputs:
-        artifactName: pymanager
-        targetPath: $(Build.BinariesDirectory)
-        buildType: specific
-        project: $(System.TeamProject)
-        pipeline: $(Build.DefinitionName)
-        buildVersionToDownload: specific
-        buildId: ${{ parameters.BuildToPublish }}
+    - ${{ each Name in variables.artifacts }}:
+      - task: DownloadPipelineArtifact@1
+        displayName: 'Download artifact from ${{ parameters.BuildToPublish}}: pymanager_${{ Name }}'
+        inputs:
+          artifactName: pymanager_${{ Name }}
+          targetPath: $(Pipeline.Workspace)
+          buildType: specific
+          project: $(System.TeamProject)
+          pipeline: $(Build.DefinitionName)
+          buildVersionToDownload: specific
+          buildId: ${{ parameters.BuildToPublish }}
 
   - ${{ else }}:
-    - task: DownloadPipelineArtifact@1
-      displayName: 'Download artifact: pymanager'
-      inputs:
-        artifactName: pymanager
-        targetPath: $(Build.BinariesDirectory)
-
-  # Note that ARM64 MSIs are skipped at build when this option is specified
-  - powershell: 'gci *embed-arm*.zip | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del'
-    displayName: 'Prevent publishing ARM64 packages'
-    workingDirectory: '$(Build.BinariesDirectory)\pymanager'
-    condition: and(succeeded(), ne(variables['PublishARM64'], 'true'))
+    - ${{ each Name in variables.artifacts }}:
+      - download: current
+        artifact: pymanager_${{ Name }}
+        displayName: 'Download artifact: pymanager_${{ Name }}'
 
 
   - task: DownloadSecureFile@1
@@ -56,25 +49,24 @@ jobs:
     displayName: 'Download PuTTY key'
 
   - powershell: |
-      "Uploading following files:"
-      dir *
+      "Uploading following packages:"
+      dir "*\__install__.json"
       python merge-and-upload.py 
-    workingDirectory: $(Build.BinariesDirectory)
+    workingDirectory: $(Pipeline.Workspace)
     displayName: 'Upload ZIPs'
     env:
-      UPLOAD_URL_PREFIX: 'https://www.python.org/ftp/'
-      UPLOAD_PATH_PREFIX: '/srv/www.python.org/ftp/'
-      INDEX_URL: 'https://www.python.org/ftp/python/__index_windows__.json'
-      INDEX_PATH: ${{ parameters.FeedPath }}
+      INDEX_URL: '$(PyDotOrgUrlPrefix)python/__index_windows__.json'
       INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\__index__.json'
+      UPLOAD_URL_PREFIX: $(PyDotOrgUrlPrefix)
+      UPLOAD_PATH_PREFIX: $(PyDotOrgUploadPathPrefix)
       UPLOAD_HOST: $(PyDotOrgServer)
       UPLOAD_HOST_KEY: $(PyDotOrgHostKey)
-      UPLOAD_KEYFILE: $(sshkey.secureFilePath)
       UPLOAD_USER: $(PyDotOrgUsername)
+      UPLOAD_KEYFILE: $(sshkey.secureFilePath)
 
   - ${{ each alg in parameters.HashAlgorithms }}:
     - powershell: |
-        $files = gci -File "*.zip"
+        $files = (dir "*\__install__.json").Directory | %{ dir -File "$_\*.zip" }
         $hashes = $files  | `
             Sort-Object Name | `
             Format-Table Name, @{
@@ -85,7 +77,7 @@ jobs:
         $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force
         $hashes | Out-File "$d\hashes.txt" -Encoding ascii -Append
         $hashes
-      workingDirectory: $(Build.BinariesDirectory)
+      workingDirectory: $(Pipeline.Workspace)
       displayName: 'Generate hashes (${{ alg }})'
 
   - publish: '$(Build.ArtifactStagingDirectory)\index'