Skip to content

Commit 0f27643

Browse files
zoobazooba
committed
[WIP] Adding example build steps for PyManager packages
1 parent 6922a3c commit 0f27643

5 files changed

Lines changed: 406 additions & 0 deletions

File tree

‎windows-release/azure-pipelines.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,10 @@ parameters:
8888
displayName: "Produce EXE/MSI installer"
8989
type: boolean
9090
default: true
91+
- name: DoPyManager
92+
displayName: "Produce PyManager package"
93+
type: boolean
94+
default: true
9195
- name: BuildToPublish
9296
displayName: "Build number to publish (0 to skip)"
9397
type: number
@@ -120,6 +124,7 @@ variables:
120124
DoNuget: ${{ parameters.DoNuget }}
121125
DoEmbed: ${{ parameters.DoEmbed }}
122126
DoMSI: ${{ parameters.DoMSI }}
127+
DoPyManager: ${{ parameters.DoPyManager }}
123128
DoPublish: ${{ parameters.DoPublish }}
124129
PublishARM64: ${{ parameters.DoARM64 }}
125130
# QUEUE TIME VARIABLES
@@ -163,6 +168,11 @@ stages:
163168
- template: stage-layout-nuget.yml
164169
parameters:
165170
DoFreethreaded: ${{ parameters.DoFreethreaded }}
171+
- template: stage-layout-pymanager.yml
172+
parameters:
173+
ARM64TclTk: ${{ parameters.ARM64TclTk }}
174+
DoFreethreaded: ${{ parameters.DoFreethreaded }}
175+
DoEmbed: ${{ parameters.DoEmbed }}
166176

167177
- stage: Pack
168178
dependsOn: Layout
@@ -173,6 +183,12 @@ stages:
173183
${{ if eq(parameters.SignNuget, 'true') }}:
174184
SigningCertificate: ${{ parameters.SigningCertificate }}
175185
DoFreethreaded: ${{ parameters.DoFreethreaded }}
186+
- template: stage-pack-pymanager.yml
187+
parameters:
188+
${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
189+
SigningCertificate: ${{ parameters.SigningCertificate }}
190+
DoFreethreaded: ${{ parameters.DoFreethreaded }}
191+
DoEmbed: ${{ parameters.DoEmbed }}
176192

177193
- stage: Test
178194
dependsOn: Pack
@@ -222,6 +238,13 @@ stages:
222238
DoFreethreaded: ${{ parameters.DoFreethreaded }}
223239

224240
- ${{ if eq(parameters.DoPublish, 'true') }}:
241+
- ${{ if eq(parameters.DoPyManager, 'true') }}:
242+
- stage: PublishPyManager
243+
displayName: Publish for PyManager
244+
dependsOn: ['Test_MSI', 'Test']
245+
jobs:
246+
- template: stage-publish-pymanager.yml
247+
225248
- ${{ if eq(parameters.DoMSI, 'true') }}:
226249
- stage: PublishPyDotOrg
227250
displayName: Publish to python.org
@@ -256,6 +279,9 @@ stages:
256279
displayName: Publish existing build
257280
dependsOn: []
258281
jobs:
282+
- ${{ if eq(parameters.DoPyManager, 'true') }}:
283+
- template: stage-publish-pymanager.yml
284+
259285
- ${{ if eq(parameters.DoMSI, 'true') }}:
260286
- template: stage-publish-pythonorg.yml
261287
parameters:

‎windows-release/merge-and-upload.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import hashlib
2+
import json
3+
import os
4+
import subprocess
5+
6+
from pathlib import Path
7+
from urllib.parse import urlparse
8+
from urllib.request import urlopen, Request
9+
10+
INDEX_PATH = os.getenv("INDEX_PATH", "ftp/downloads/__index_windows__.json")
11+
INDEX_URL = os.getenv("INDEX_URL") or f"https://www.python.org/{INDEX_PATH}"
12+
UPLOAD_HOST = os.getenv("PyDotOrgServer")
13+
14+
15+
def call_ssh(args):
16+
if not UPLOAD_HOST:
17+
print("Skipping", args, "because UPLOAD_HOST is missing")
18+
return
19+
subprocess.check_output(args)
20+
21+
22+
def get_hashes(src):
23+
h = hashlib.sha256()
24+
with open(src, "rb") as f:
25+
chunk = f.read(1024 * 1024)
26+
while chunk:
27+
h.update(chunk)
28+
chunk = f.read(1024 * 1024)
29+
return {"sha256": h.hexdigest()}
30+
31+
32+
def purge(url):
33+
if not UPLOAD_HOST:
34+
print("Skipping purge of", url, "because UPLOAD_HOST is missing")
35+
return
36+
with urlopen(Request(url, method="PURGE", headers={"Fastly-Soft-Purge": 1})) as r:
37+
r.read()
38+
39+
40+
def calculate_uploads():
41+
for p in Path().absolute().glob("__install*.json"):
42+
i = json.loads(p.read_bytes())
43+
u = urlparse(i["url"])
44+
yield (
45+
i,
46+
Path(u.path.rpartition("/")[-1]).absolute(),
47+
u.path,
48+
)
49+
50+
def upload_files(uploads):
51+
for i, src, dest in uploads:
52+
print("Uploading", src, "to", dest)
53+
call_ssh([...])
54+
55+
def purge_files(uploads):
56+
for i, src, dest in uploads:
57+
purge(i["url"])
58+
59+
def hash_packages(uploads):
60+
for i, src, dest in uploads:
61+
i["hashes"] = get_hashes(src)
62+
63+
64+
UPLOADS = list(calculate_uploads())
65+
hash_packages(UPLOADS)
66+
67+
68+
try:
69+
with open("__index__.json", "rb") as f:
70+
index = json.load(f)
71+
except FileNotFoundError:
72+
index = {"versions": []}
73+
74+
75+
# TODO: Sort?
76+
index["versions"][:0] = [i[0] for i in UPLOADS]
77+
78+
with open("__index__.json", "wb") as f:
79+
# Include an indent for sanity while testing.
80+
# We should probably remove it later for the size benefits.
81+
json.dump(f, index, indent=1)
82+
83+
print("Merged", len(UPLOADS), "entries")
84+
85+
# Upload last to ensure we've got a valid index first
86+
upload_files(UPLOADS)
87+
88+
print("Uploading __index__.json to python.org")
89+
run_ssh([...])
90+
91+
purge_files(UPLOADS)
92+
purge(INDEX_URL)

‎windows-release/stage-layout-pymanager.yml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
parameters:
2+
ARM64TclTk: true
3+
DoFreethreaded: false
4+
DoEmbed: false
5+
6+
jobs:
7+
- job: Make_PyManager_Layouts
8+
displayName: Make PyManager layouts
9+
condition: and(succeeded(), eq(variables['DoPyManager'], 'true'))
10+
11+
pool:
12+
vmImage: windows-2022
13+
14+
workspace:
15+
clean: all
16+
17+
variables:
18+
PYTHONHOME: $(Build.SourcesDirectory)
19+
20+
strategy:
21+
matrix:
22+
win32:
23+
Name: win32
24+
Arch: win32
25+
TclLibrary: tcltk_lib_win32\tcl8
26+
LayoutOptions: '--preset-pymanager'
27+
IncludeDoc: true
28+
amd64:
29+
Name: amd64
30+
Arch: amd64
31+
TclLibrary: tcltk_lib_amd64\tcl8
32+
LayoutOptions: '--preset-pymanager'
33+
IncludeDoc: true
34+
arm64:
35+
Name: arm64
36+
Arch: arm64
37+
HostArch: amd64
38+
LayoutOptions: '--preset-pymanager'
39+
IncludeDoc: true
40+
${{ if eq(parameters.ARM64TclTk, 'true') }}:
41+
TclLibrary: tcltk_lib_arm64\tcl8
42+
${{ if eq(parameters.DoFreethreaded, 'true') }}:
43+
win32_t:
44+
Name: win32_t
45+
Arch: win32
46+
HostArch: win32
47+
TclLibrary: tcltk_lib_win32\tcl8
48+
LayoutOptions: '--preset-pymanager --include-freethreaded'
49+
IncludeDoc: true
50+
amd64_t:
51+
Name: amd64_t
52+
Arch: amd64
53+
HostArch: amd64
54+
TclLibrary: tcltk_lib_amd64\tcl8
55+
LayoutOptions: '--preset-pymanager --include-freethreaded'
56+
IncludeDoc: true
57+
arm64_t:
58+
Name: arm64_t
59+
Arch: arm64
60+
HostArch: amd64
61+
TclLibrary: tcltk_lib_arm64\tcl8
62+
LayoutOptions: '--preset-pymanager --include-freethreaded'
63+
IncludeDoc: true
64+
${{ if eq(parameters.DoEmbed, 'true') }}:
65+
win32_embed:
66+
Name: win32_embed
67+
Arch: win32
68+
HostArch: win32
69+
LayoutOptions: '--preset-embed --include-install-embed-json'
70+
amd64_embed:
71+
Name: amd64_embed
72+
Arch: amd64
73+
HostArch: amd64
74+
LayoutOptions: '--preset-embed --include-install-embed-json'
75+
arm64_embed:
76+
Name: arm64_embed
77+
Arch: arm64
78+
HostArch: arm64
79+
LayoutOptions: '--preset-embed --include-install-embed-json'
80+
81+
steps:
82+
- template: ./checkout.yml
83+
84+
- download: current
85+
artifact: bin_$(Name)
86+
displayName: 'Download artifact: bin_$(Name)'
87+
88+
- download: current
89+
artifact: doc
90+
displayName: 'Download artifact: doc'
91+
condition: and(succeeded(), variables['IncludeDoc'])
92+
93+
- download: current
94+
artifact: $(TclLibrary)
95+
displayName: 'Download artifact: $(TclLibrary)'
96+
condition: and(succeeded(), variables['TclLibrary'])
97+
98+
- powershell: |
99+
Write-Host "##vso[task.setvariable variable=TCL_LIBRARY]$(Pipeline.Workspace)\$(TclLibrary)"
100+
displayName: 'Update TCL_LIBRARY'
101+
condition: and(succeeded(), variables['TclLibrary'])
102+
103+
- powershell: |
104+
copy "$(Pipeline.Workspace)\bin_$(Name)\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force
105+
displayName: 'Copy signed files into sources'
106+
condition: and(succeeded(), variables['SigningCertificate'])
107+
108+
- template: ./layout-command.yml
109+
110+
- powershell: |
111+
$(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" $(LayoutOptions)
112+
displayName: 'Generate PyManager layout'
113+
114+
- publish: '$(Build.ArtifactStagingDirectory)\layout'
115+
artifact: layout_pymanager_$(Name)
116+
displayName: 'Publish Artifact: layout_pymanager_$(Name)'

‎windows-release/stage-pack-pymanager.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
parameters:
2+
DoFreethreaded: false
3+
DoEmbed: false
4+
SigningCertificate: ''
5+
6+
jobs:
7+
- job: Pack_PyManager
8+
displayName: Pack PyManager bundles
9+
condition: and(succeeded(), eq(variables['DoPyManager'], 'true'))
10+
11+
pool:
12+
vmImage: windows-2022
13+
14+
workspace:
15+
clean: all
16+
17+
variables:
18+
19+
variables:
20+
#- ${{ if eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}:
21+
# - group: CPythonSign
22+
#- ${{ if eq(parameters.SigningCertificate, 'TestSign') }}:
23+
# - group: CPythonTestSign
24+
- name: artifacts
25+
value:
26+
- amd64
27+
- win32
28+
- arm64
29+
- ${{ if eq(parameters.DoEmbed, 'true') }}:
30+
- amd64_embed
31+
- win32_embed
32+
- arm64_embed
33+
- ${{ if eq(parameters.DoFreethreaded, 'true') }}:
34+
- amd64_t
35+
- win32_t
36+
- arm64_t
37+
38+
steps:
39+
- checkout: none
40+
41+
- ${{ each Name in variables.artifacts }}:
42+
- download: current
43+
artifact: layout_pymanager_${{ Name }}
44+
displayName: 'Download artifact: layout_pymanager_${{ Name }}'
45+
46+
- ${{ each Name in variables.artifacts }}:
47+
- powershell: |
48+
$install = (gc -raw "${env:LAYOUT}\__install__.json") -replace '"":', '"_":' | ConvertFrom-Json
49+
$filename = Split-Path -Left $install.url
50+
# Deliberately move the __install__.json out. We need it to publish, but
51+
# it doesn't belong in the package.
52+
mv $env:INSTALL_JSON $env:INSTALL_JSON
53+
Compress-Archive -CompressionLevel Optimal $env:LAYOUT "$(Build.ArtifactStagingDirectory)\$filename"
54+
env:
55+
LAYOUT: $(Pipeline.Workspace)\layout_pymanager_${{ Name }}
56+
INSTALL_JSON: $(Build.ArtifactStagingDirectory)\__install__.${{ Name }}.json
57+
58+
- publish: '$(Build.ArtifactStagingDirectory)'
59+
artifact: pymanager
60+
displayName: 'Publish Artifact: pymanager'

0 commit comments

Comments
 (0)