From f3828e44db27a6ae3e61b6cbdf712047d5a234be Mon Sep 17 00:00:00 2001 From: MaxHearnden Date: Tue, 10 Sep 2024 05:57:49 +0100 Subject: [PATCH 1/8] Add minimal unprivileged bootstrap --- lib/generator.py | 9 ++++++--- rootfs.py | 19 +++++++++++++++---- seed/after-wrap.kaem | 19 +++++++++++++++++++ seed/after.kaem | 15 ++++++++++++++- seed/seed.kaem | 7 ++++++- seed/wrap-bootstrap.cfg | 5 +++++ steps/improve/setup_repo.sh | 2 +- 7 files changed, 66 insertions(+), 10 deletions(-) create mode 100644 seed/after-wrap.kaem create mode 100644 seed/wrap-bootstrap.cfg diff --git a/lib/generator.py b/lib/generator.py index 2f69058b..8ccaafbb 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -42,7 +42,7 @@ def reuse(self, target): self.external_dir = os.path.join(self.target_dir, 'external') self.distfiles() - def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0): + def prepare(self, target, using_kernel=False, kernel_bootstrap=False, wrap=False, target_size=0): """ Prepare basic media of live-bootstrap. /steps -- contains steps to be built @@ -82,7 +82,7 @@ def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_siz os.path.join(self.target_dir, 'kaem.x86')) else: self.stage0_posix(kernel_bootstrap) - self.seed() + self.seed(wrap) os.makedirs(self.external_dir) @@ -134,12 +134,15 @@ def stage0_posix(self, kernel_bootstrap=False): 'kaem-optional-seed') shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init')) - def seed(self): + def seed(self, wrap): """Copy in extra seed files""" seed_dir = os.path.join(self.git_dir, 'seed') for entry in os.listdir(seed_dir): if os.path.isfile(os.path.join(seed_dir, entry)): shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry)) + if wrap: + shutil.copy2(os.path.join(seed_dir, 'after-wrap.kaem'), os.path.join(self.target_dir, 'after.kaem')) + shutil.copy2(os.path.join(seed_dir, 'after.kaem'), os.path.join(self.target_dir, 'after-wrapped.kaem')) def distfiles(self): """Copy in distfiles""" diff --git a/rootfs.py b/rootfs.py index 1cfeab94..a68e5998 100755 --- a/rootfs.py +++ b/rootfs.py @@ -31,7 +31,7 @@ def create_configuration_file(args): config.write(f"ARCH={args.arch}\n") config.write(f"ARCH_DIR={stage0_arch_map.get(args.arch, args.arch)}\n") config.write(f"FORCE_TIMESTAMPS={args.force_timestamps}\n") - config.write(f"CHROOT={args.chroot or args.bwrap}\n") + config.write(f"CHROOT={args.chroot or args.bwrap or args.wrap}\n") config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n") config.write(f"JOBS={args.cores}\n") config.write(f"SWAP_SIZE={args.swap}\n") @@ -63,6 +63,8 @@ def main(): default="x86") parser.add_argument("-c", "--chroot", help="Run inside chroot", action="store_true") + parser.add_argument("-w", "--wrap", help="Run inside a minimal sandbox", + action="store_true") parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox", action="store_true") parser.add_argument("-t", "--target", help="Target directory", @@ -127,15 +129,18 @@ def check_types(): count += 1 if args.bwrap: count += 1 + if args.wrap: + count += 1 if args.bare_metal: count += 1 return count if check_types() > 1: - raise ValueError("No more than one of qemu, chroot, bwrap, bare metal" - "may be used.") + raise ValueError("No more than one of qemu, chroot, bwrap, wrap, bare " + "metal may be used.") if check_types() == 0: - raise ValueError("One of qemu, chroot, bwrap, or bare metal must be selected.") + raise ValueError("One of qemu, chroot, bwrap, wrap, or bare metal must" + " be selected.") # Arch validation if args.arch != "x86": @@ -237,6 +242,12 @@ def bootstrap(args, generator, target, size): '--tmpfs', '/tmp', init) + elif args.wrap: + generator.prepare(target, wrap = True) + arch = stage0_arch_map.get(args.arch, args.arch) + init = os.path.join('bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') + run(init, cwd = generator.target_dir) + elif args.bare_metal: if args.kernel: generator.prepare(target, using_kernel=True, target_size=size) diff --git a/seed/after-wrap.kaem b/seed/after-wrap.kaem new file mode 100644 index 00000000..ebbfef4b --- /dev/null +++ b/seed/after-wrap.kaem @@ -0,0 +1,19 @@ +#!/bin/sh + +# SPDX-FileCopyrightText: 2024 Max Hearnden maxoscarhearnden@gmail.com +# +# SPDX-License-Identifier: GPL-3.0-or-later + +set -ex + +# detect wether we are in a rootfs.py environment +if ./${ARCH_DIR}/bin/catm steps/env-saved steps/env; then + ./${ARCH_DIR}/bin/wrap /${ARCH_DIR}/bin/kaem --file after-wrapped.kaem +else + # leave seed/stage0-posix + cd ../.. + + ARCH_DIR=seed/stage0-posix/${ARCH_DIR} + + ./${ARCH_DIR}/bin/wrap /${ARCH_DIR}/bin/kaem --file seed/after.kaem +fi diff --git a/seed/after.kaem b/seed/after.kaem index a9ab9aa0..b9dae1e3 100755 --- a/seed/after.kaem +++ b/seed/after.kaem @@ -11,5 +11,18 @@ set -ex PATH=/${ARCH_DIR}/bin -catm seed-full.kaem /steps/bootstrap.cfg /steps/env seed.kaem +if catm seed-full.kaem /steps/bootstrap.cfg /steps/env seed.kaem; then +else + replace --file /steps/env --output /steps/env --match-on /external/distfiles --replace-with /distfiles + cp /seed/wrap-bootstrap.cfg /steps/bootstrap.cfg + catm seed-full.kaem /steps/bootstrap.cfg /steps/env /seed/seed.kaem + cp /seed/configurator.c configurator.c + cp /seed/configurator.${ARCH}.checksums configurator.${ARCH}.checksums + + cp /seed/script-generator.c script-generator.c + cp /seed/script-generator.${ARCH}.checksums script-generator.${ARCH}.checksums + # placeholder value + FINAL_JOBS=1 +fi + kaem --file seed-full.kaem diff --git a/seed/seed.kaem b/seed/seed.kaem index 1468cc1d..d56c480c 100755 --- a/seed/seed.kaem +++ b/seed/seed.kaem @@ -64,7 +64,12 @@ MES_PKG=mes-0.27 MES_PREFIX=${SRCDIR}/${MES_PKG}/build/${MES_PKG} GUILE_LOAD_PATH=${MES_PREFIX}/mes/module:${MES_PREFIX}/module:${SRCDIR}/${MES_PKG}/build/${NYACC_PKG}/module -M2-Mesoplanet --architecture ${ARCH} -f configurator.c -o configurator +if M2-Mesoplanet --architecture ${ARCH} -f configurator.c -o configurator; then +else + # using lightweight wrapper + M2LIBC_PATH=/seed/stage0-posix/M2libc + M2-Mesoplanet --architecture ${ARCH} -f configurator.c -o configurator +fi # Checksums if match x${UPDATE_CHECKSUMS} xTrue; then sha256sum -o configurator.${ARCH}.checksums configurator diff --git a/seed/wrap-bootstrap.cfg b/seed/wrap-bootstrap.cfg new file mode 100644 index 00000000..6a827cad --- /dev/null +++ b/seed/wrap-bootstrap.cfg @@ -0,0 +1,5 @@ +CHROOT=True +DISK=sda1 +KERNEL_BOOTSTRAP=False +BUILD_KERNELS=False +JOBS=${FINAL_JOBS} diff --git a/steps/improve/setup_repo.sh b/steps/improve/setup_repo.sh index 60fc3512..cabf07a6 100755 --- a/steps/improve/setup_repo.sh +++ b/steps/improve/setup_repo.sh @@ -6,4 +6,4 @@ # mkdir -p /external/repo -tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2 +tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2 From 4e2434b837d6c9042cf6682f42da4fe2def6e81f Mon Sep 17 00:00:00 2001 From: MaxHearnden Date: Tue, 10 Sep 2024 15:42:11 +0100 Subject: [PATCH 2/8] Support building within git repository The lib directory for python modules conflicted with the symlinks for a merged usr so this moves the directory out of the way --- steps/improve/merged_usr.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/steps/improve/merged_usr.sh b/steps/improve/merged_usr.sh index 14e79d87..33d30748 100755 --- a/steps/improve/merged_usr.sh +++ b/steps/improve/merged_usr.sh @@ -6,5 +6,9 @@ # Add the rest of the FHS that we will use and is not created pre-boot ln -s bin /usr/sbin for d in bin lib sbin; do + if [ -d "/${d}" ] && ! [ -L "/${d}" ]; then + # Move the non symlink directory out of the way + mv "/${d}" "/${d}-saved" + fi ln -s "usr/${d}" "/${d}" || true # these might exist if rerunning done From cdd1114dc5cbbc0ba88d9501fa7e524fc26e71c1 Mon Sep 17 00:00:00 2001 From: MaxHearnden Date: Wed, 11 Sep 2024 20:43:36 +0100 Subject: [PATCH 3/8] add reuse information for seed/wrap-bootstrap.cfg --- .reuse/dep5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.reuse/dep5 b/.reuse/dep5 index 0b3e3bd6..2f6bf563 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -9,6 +9,6 @@ Source: https://github.com/fosslinux/live-bootstrap # Copyright: $YEAR $NAME <$CONTACT> # License: ... -Files: steps/*/sources steps/*/*.checksums steps/SHA256SUMS.pkgs steps/*/simple-patches/* seed/*.checksums +Files: steps/*/sources steps/*/*.checksums steps/SHA256SUMS.pkgs steps/*/simple-patches/* seed/*.checksums seed/wrap-bootstrap.cfg Copyright: none License: MIT From fc2c576dde769493fe6a278d9d42b2c343d4ffa0 Mon Sep 17 00:00:00 2001 From: MaxHearnden Date: Fri, 13 Sep 2024 10:44:45 +0100 Subject: [PATCH 4/8] Add CI workflow for wrap and fix lint warnings --- .github/workflows/wrap.yml | 153 +++++++++++++++++++++++++++++++++++++ lib/generator.py | 14 +++- rootfs.py | 11 ++- 3 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/wrap.yml diff --git a/.github/workflows/wrap.yml b/.github/workflows/wrap.yml new file mode 100644 index 00000000..91f71c69 --- /dev/null +++ b/.github/workflows/wrap.yml @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: 2023 fosslinux +# SPDX-FileCopyrightText: 2024 Max Hearnden +# +# SPDX-License-Identifier: GPL-3.0-or-later + +name: Run under wrap + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + +jobs: + pass1: + name: Run up to Linux build under wrap + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + submodules: recursive + # There is a strange bug(?) in nongnu, when you clone a git repository + # against a commit != HEAD with depth=1, it errors out. + fetch-depth: 0 + - name: Query cache for sources + id: cache + uses: actions/cache/restore@v3 + with: + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} + - name: Get sources + if: steps.cache.outputs.cache-hit != 'true' + run: ./download-distfiles.sh + - name: Cache sources + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} + - name: Run bootstrap + run: ./rootfs.py --wrap --external-sources --build-kernels --cores 2 --internal-ci pass1 + - name: Archive created packages + if: failure() # archive failed builds progress + uses: actions/upload-artifact@v3 + with: + name: packages + path: target/external/repo/** + - name: Tar pass1 image + run: tar -cf pass1_image.tar target + - name: Archive pass1_image + uses: actions/upload-artifact@v3 + with: + name: internal_pass1_image + path: pass1_image.tar + + pass2: + name: Run up to Python bootstrap under wrap + needs: pass1 + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + submodules: recursive + # There is a strange bug(?) in nongnu, when you clone a git repository + # against a commit != HEAD with depth=1, it errors out. + fetch-depth: 0 + - name: Get pass1_image + uses: actions/download-artifact@v3 + with: + name: internal_pass1_image + - name: Extract pass1_image + run: tar -xf pass1_image.tar + - name: Query cache for sources + id: cache + uses: actions/cache/restore@v3 + with: + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} + - name: Get sources + if: steps.cache.outputs.cache-hit != 'true' + run: ./download-distfiles.sh + - name: Cache sources + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} + - name: Run bootstrap + run: ./rootfs.py --wrap --external-sources --build-kernels --cores 2 --internal-ci pass2 + - name: Archive created packages + if: failure() # archive failed builds progress + uses: actions/upload-artifact@v3 + with: + name: internal_packages_pass2 + path: target/external/repo/** + - name: Tar pass2 image + run: tar -cf pass2_image.tar target + - name: Archive pass2_image + uses: actions/upload-artifact@v3 + with: + name: internal_pass2_image + path: pass2_image.tar + + pass3: + name: Run remaining builds under wrap + needs: pass2 + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + submodules: recursive + # There is a strange bug(?) in nongnu, when you clone a git repository + # against a commit != HEAD with depth=1, it errors out. + fetch-depth: 0 + - name: Get pass2_image + uses: actions/download-artifact@v3 + with: + name: internal_pass2_image + - name: Extract pass2_image + run: tar -xf pass2_image.tar + - name: Query cache for sources + id: cache + uses: actions/cache/restore@v3 + with: + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} + - name: Get sources + if: steps.cache.outputs.cache-hit != 'true' + run: ./download-distfiles.sh + - name: Cache sources + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} + - name: Run bootstrap + run: ./rootfs.py --wrap --external-sources --build-kernels --cores 2 --internal-ci pass3 + - name: Archive created packages + if: always() # archive both failed and successful builds + uses: actions/upload-artifact@v3 + with: + name: packages + path: target/external/repo/** diff --git a/lib/generator.py b/lib/generator.py index 8ccaafbb..1e41f9d3 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -42,7 +42,13 @@ def reuse(self, target): self.external_dir = os.path.join(self.target_dir, 'external') self.distfiles() - def prepare(self, target, using_kernel=False, kernel_bootstrap=False, wrap=False, target_size=0): + # pylint: disable=too-many-arguments + def prepare(self, + target, + using_kernel=False, + kernel_bootstrap=False, + wrap=False, + target_size=0): """ Prepare basic media of live-bootstrap. /steps -- contains steps to be built @@ -141,8 +147,10 @@ def seed(self, wrap): if os.path.isfile(os.path.join(seed_dir, entry)): shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry)) if wrap: - shutil.copy2(os.path.join(seed_dir, 'after-wrap.kaem'), os.path.join(self.target_dir, 'after.kaem')) - shutil.copy2(os.path.join(seed_dir, 'after.kaem'), os.path.join(self.target_dir, 'after-wrapped.kaem')) + shutil.copy2(os.path.join(seed_dir, 'after-wrap.kaem'), + os.path.join(self.target_dir, 'after.kaem')) + shutil.copy2(os.path.join(seed_dir, 'after.kaem'), + os.path.join(self.target_dir, 'after-wrapped.kaem')) def distfiles(self): """Copy in distfiles""" diff --git a/rootfs.py b/rootfs.py index a68e5998..87fb29d2 100755 --- a/rootfs.py +++ b/rootfs.py @@ -243,10 +243,15 @@ def bootstrap(args, generator, target, size): init) elif args.wrap: - generator.prepare(target, wrap = True) arch = stage0_arch_map.get(args.arch, args.arch) - init = os.path.join('bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - run(init, cwd = generator.target_dir) + if not args.internal_ci or args.internal_ci == "pass1": + generator.prepare(target, wrap = True) + arg_list = [os.path.join('bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')] + else: + generator.reuse(target) + arg_list = [os.path.join(arch, 'bin', 'wrap'), '/init'] + + run(*arg_list, cwd = generator.target_dir) elif args.bare_metal: if args.kernel: From dc5a00c94f244055d37ea0a34884e1671d1b604d Mon Sep 17 00:00:00 2001 From: MaxHearnden Date: Sat, 14 Sep 2024 15:09:32 +0100 Subject: [PATCH 5/8] use a test matrix --- .github/workflows/bwrap.yml | 39 +++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index 66cea846..ee484472 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -name: Run under bubblewrap +name: Run under (bubble)wrap on: workflow_dispatch: @@ -13,10 +13,15 @@ on: jobs: pass1: - name: Run up to Linux build under bubblewrap + name: Run up to Linux build under (bubble)wrap runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + virt-tool: [bwrap, wrap] steps: - name: Install bubblewrap + if: matrix.virt-tool == 'bwrap' run: sudo apt install bubblewrap - name: Checkout repo uses: actions/checkout@v3 @@ -43,7 +48,7 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1 + run: ./rootfs.py --${{matrix.virt-tool}} --external-sources --build-kernels --cores 2 --internal-ci pass1 - name: Archive created packages if: failure() # archive failed builds progress uses: actions/upload-artifact@v3 @@ -56,14 +61,19 @@ jobs: uses: actions/upload-artifact@v3 with: name: internal_pass1_image - path: pass1_image.tar + path: pass1_image_${{matrix.virt-tool}}.tar pass2: - name: Run up to Python bootstrap under bubblewrap + name: Run up to Python bootstrap under (bubble)wrap needs: pass1 runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + virt-tool: [bwrap, wrap] steps: - name: Install bubblewrap + if: matrix.virt-tool == 'bwrap' run: sudo apt install bubblewrap - name: Checkout repo uses: actions/checkout@v3 @@ -77,7 +87,7 @@ jobs: with: name: internal_pass1_image - name: Extract pass1_image - run: tar -xf pass1_image.tar + run: tar -xf pass1_image_${{matrix.virt-tool}}.tar - name: Query cache for sources id: cache uses: actions/cache/restore@v3 @@ -96,7 +106,7 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2 + run: ./rootfs.py --${{matrix.virt-tool}} --external-sources --build-kernels --cores 2 --internal-ci pass2 - name: Archive created packages if: failure() # archive failed builds progress uses: actions/upload-artifact@v3 @@ -109,14 +119,19 @@ jobs: uses: actions/upload-artifact@v3 with: name: internal_pass2_image - path: pass2_image.tar + path: pass2_image_${{matrix.virt-tool}}.tar pass3: - name: Run remaining builds under bubblewrap + name: Run remaining builds under (bubble)wrap needs: pass2 runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + virt-tool: [bwrap, wrap] steps: - name: Install bubblewrap + if: matrix.virt-tool == 'bwrap' run: sudo apt install bubblewrap - name: Checkout repo uses: actions/checkout@v3 @@ -130,7 +145,7 @@ jobs: with: name: internal_pass2_image - name: Extract pass2_image - run: tar -xf pass2_image.tar + run: tar -xf pass2_image_${{matrix.virt-tool}}.tar - name: Query cache for sources id: cache uses: actions/cache/restore@v3 @@ -149,10 +164,10 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 + run: ./rootfs.py --${{matrix.virt-tool}} --external-sources --build-kernels --cores 2 --internal-ci pass3 - name: Archive created packages if: always() # archive both failed and successful builds uses: actions/upload-artifact@v3 with: - name: packages + name: packages_${{matrix.virt-tool}} path: target/external/repo/** From fa7a7ea1400e79138f252909ba28a96b20098c62 Mon Sep 17 00:00:00 2001 From: MaxHearnden Date: Sat, 14 Sep 2024 17:06:06 +0100 Subject: [PATCH 6/8] fix archive names --- .github/workflows/bwrap.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index ee484472..005fd992 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -56,7 +56,7 @@ jobs: name: packages path: target/external/repo/** - name: Tar pass1 image - run: tar -cf pass1_image.tar target + run: tar -cf pass1_image_${{matrix.virt-tool}}.tar target - name: Archive pass1_image uses: actions/upload-artifact@v3 with: @@ -114,7 +114,7 @@ jobs: name: internal_packages_pass2 path: target/external/repo/** - name: Tar pass2 image - run: tar -cf pass2_image.tar target + run: tar -cf pass2_image_${{matrix.virt-tool}}.tar target - name: Archive pass2_image uses: actions/upload-artifact@v3 with: From edcaf06ab39a4358b418de298fc1cea54e7e7b5e Mon Sep 17 00:00:00 2001 From: MaxHearnden Date: Sat, 14 Sep 2024 17:09:49 +0100 Subject: [PATCH 7/8] remove wrap.yml --- .github/workflows/wrap.yml | 153 ------------------------------------- 1 file changed, 153 deletions(-) delete mode 100644 .github/workflows/wrap.yml diff --git a/.github/workflows/wrap.yml b/.github/workflows/wrap.yml deleted file mode 100644 index 91f71c69..00000000 --- a/.github/workflows/wrap.yml +++ /dev/null @@ -1,153 +0,0 @@ -# SPDX-FileCopyrightText: 2023 fosslinux -# SPDX-FileCopyrightText: 2024 Max Hearnden -# -# SPDX-License-Identifier: GPL-3.0-or-later - -name: Run under wrap - -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - -jobs: - pass1: - name: Run up to Linux build under wrap - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v3 - with: - submodules: recursive - # There is a strange bug(?) in nongnu, when you clone a git repository - # against a commit != HEAD with depth=1, it errors out. - fetch-depth: 0 - - name: Query cache for sources - id: cache - uses: actions/cache/restore@v3 - with: - path: | - distfiles - key: cache-${{ hashFiles('steps/*/sources') }} - - name: Get sources - if: steps.cache.outputs.cache-hit != 'true' - run: ./download-distfiles.sh - - name: Cache sources - if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v3 - with: - path: | - distfiles - key: cache-${{ hashFiles('steps/*/sources') }} - - name: Run bootstrap - run: ./rootfs.py --wrap --external-sources --build-kernels --cores 2 --internal-ci pass1 - - name: Archive created packages - if: failure() # archive failed builds progress - uses: actions/upload-artifact@v3 - with: - name: packages - path: target/external/repo/** - - name: Tar pass1 image - run: tar -cf pass1_image.tar target - - name: Archive pass1_image - uses: actions/upload-artifact@v3 - with: - name: internal_pass1_image - path: pass1_image.tar - - pass2: - name: Run up to Python bootstrap under wrap - needs: pass1 - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v3 - with: - submodules: recursive - # There is a strange bug(?) in nongnu, when you clone a git repository - # against a commit != HEAD with depth=1, it errors out. - fetch-depth: 0 - - name: Get pass1_image - uses: actions/download-artifact@v3 - with: - name: internal_pass1_image - - name: Extract pass1_image - run: tar -xf pass1_image.tar - - name: Query cache for sources - id: cache - uses: actions/cache/restore@v3 - with: - path: | - distfiles - key: cache-${{ hashFiles('steps/*/sources') }} - - name: Get sources - if: steps.cache.outputs.cache-hit != 'true' - run: ./download-distfiles.sh - - name: Cache sources - if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v3 - with: - path: | - distfiles - key: cache-${{ hashFiles('steps/*/sources') }} - - name: Run bootstrap - run: ./rootfs.py --wrap --external-sources --build-kernels --cores 2 --internal-ci pass2 - - name: Archive created packages - if: failure() # archive failed builds progress - uses: actions/upload-artifact@v3 - with: - name: internal_packages_pass2 - path: target/external/repo/** - - name: Tar pass2 image - run: tar -cf pass2_image.tar target - - name: Archive pass2_image - uses: actions/upload-artifact@v3 - with: - name: internal_pass2_image - path: pass2_image.tar - - pass3: - name: Run remaining builds under wrap - needs: pass2 - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v3 - with: - submodules: recursive - # There is a strange bug(?) in nongnu, when you clone a git repository - # against a commit != HEAD with depth=1, it errors out. - fetch-depth: 0 - - name: Get pass2_image - uses: actions/download-artifact@v3 - with: - name: internal_pass2_image - - name: Extract pass2_image - run: tar -xf pass2_image.tar - - name: Query cache for sources - id: cache - uses: actions/cache/restore@v3 - with: - path: | - distfiles - key: cache-${{ hashFiles('steps/*/sources') }} - - name: Get sources - if: steps.cache.outputs.cache-hit != 'true' - run: ./download-distfiles.sh - - name: Cache sources - if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v3 - with: - path: | - distfiles - key: cache-${{ hashFiles('steps/*/sources') }} - - name: Run bootstrap - run: ./rootfs.py --wrap --external-sources --build-kernels --cores 2 --internal-ci pass3 - - name: Archive created packages - if: always() # archive both failed and successful builds - uses: actions/upload-artifact@v3 - with: - name: packages - path: target/external/repo/** From 1ae0a316eb98c6809d0ae6197d19f4b4eb9d386f Mon Sep 17 00:00:00 2001 From: MaxHearnden Date: Mon, 16 Sep 2024 16:41:10 +0100 Subject: [PATCH 8/8] use seperate artifacts for bwrap and wrap --- .github/workflows/bwrap.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index 005fd992..decdbcc4 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -56,12 +56,12 @@ jobs: name: packages path: target/external/repo/** - name: Tar pass1 image - run: tar -cf pass1_image_${{matrix.virt-tool}}.tar target + run: tar -cf pass1_image.tar target - name: Archive pass1_image uses: actions/upload-artifact@v3 with: - name: internal_pass1_image - path: pass1_image_${{matrix.virt-tool}}.tar + name: internal_pass1_image_${{matrix.virt-tool}} + path: pass1_image.tar pass2: name: Run up to Python bootstrap under (bubble)wrap @@ -85,9 +85,9 @@ jobs: - name: Get pass1_image uses: actions/download-artifact@v3 with: - name: internal_pass1_image + name: internal_pass1_image_${{matrix.virt-tool}} - name: Extract pass1_image - run: tar -xf pass1_image_${{matrix.virt-tool}}.tar + run: tar -xf pass1_image.tar - name: Query cache for sources id: cache uses: actions/cache/restore@v3 @@ -114,12 +114,12 @@ jobs: name: internal_packages_pass2 path: target/external/repo/** - name: Tar pass2 image - run: tar -cf pass2_image_${{matrix.virt-tool}}.tar target + run: tar -cf pass2_image.tar target - name: Archive pass2_image uses: actions/upload-artifact@v3 with: - name: internal_pass2_image - path: pass2_image_${{matrix.virt-tool}}.tar + name: internal_pass2_image_${{matrix.virt-tool}} + path: pass2_image.tar pass3: name: Run remaining builds under (bubble)wrap @@ -143,9 +143,9 @@ jobs: - name: Get pass2_image uses: actions/download-artifact@v3 with: - name: internal_pass2_image + name: internal_pass2_image_${{matrix.virt-tool}} - name: Extract pass2_image - run: tar -xf pass2_image_${{matrix.virt-tool}}.tar + run: tar -xf pass2_image.tar - name: Query cache for sources id: cache uses: actions/cache/restore@v3