From b08c34fc4a2d8fba248199061e5bf3cf99c3704b Mon Sep 17 00:00:00 2001 From: Benjamin Pannell Date: Mon, 1 Jun 2026 14:51:09 +0100 Subject: [PATCH] feat: back up release notes to RELEASE_NOTES.md and Forgejo release descriptions Introduce a Release entity that bundles a release's assets and notes as the unit of backup. Release notes are written to RELEASE_NOTES.md on the filesystem and replicated to the Forgejo release description, with source.tar.gz and RELEASE_NOTES.md excluded from Forgejo asset uploads. Filtering continues to operate per-asset via the source's internal filtering. --- docs/reference/release.md | 18 +++ src/engines/forgejo/release.rs | 213 +++++++++++++++++++++----------- src/engines/mod.rs | 90 ++++++++------ src/engines/release.rs | 196 +++++++++++++++++++++++++++++ src/entities/mod.rs | 2 + src/entities/release.rs | 17 +++ src/helpers/forgejo/client.rs | 25 +++- src/helpers/forgejo/entities.rs | 31 +++++ src/helpers/forgejo/mod.rs | 23 +++- src/pairing.rs | 24 ++-- src/sources/github_releases.rs | 138 +++++++++++++++++++-- src/sources/mod.rs | 12 ++ 12 files changed, 661 insertions(+), 128 deletions(-) create mode 100644 src/engines/release.rs create mode 100644 src/entities/release.rs diff --git a/docs/reference/release.md b/docs/reference/release.md index 2d4559e..bb8d3b1 100644 --- a/docs/reference/release.md +++ b/docs/reference/release.md @@ -9,6 +9,18 @@ kind in your configuration file. This kind supports the same `from` directives as the `github/repo` kind, allowing you to backup releases from your own repositories, those of other users, or those of an organization. +## Release Notes +Each release is backed up together with its release notes (the description +shown on the GitHub release page). When backing up to the local filesystem, +the notes are written to a `RELEASE_NOTES.md` file alongside the release's +artifacts (under `///`). When replicating to a Forgejo +target, the notes are stored in the corresponding Forgejo release's +description instead, and are kept in sync if they change on the source. + +The source code archive (`source.tar.gz`) and the `RELEASE_NOTES.md` file are +not uploaded as assets when replicating to Forgejo, since Forgejo generates +its own source archives and the notes are stored in the release description. + ## Examples ```yaml{5-6,11-12,16-17,22-23} title="config.yaml" @@ -43,6 +55,12 @@ When backing up release artifacts, you may use the following fields in your filt expressions. These fields are accessed using the `release.` syntax, for example `release.prerelease` to determine if a release is a pre-release. +Filters are evaluated per-asset, so the `asset.*` fields let you control exactly +which artifacts within a release are backed up, while the `release.*` and `repo.*` +fields apply to the release and its source repository as a whole. The release +notes are backed up whenever the release matches at the `release.*` / `repo.*` +level. + For `kind: github/release` | Field | Type | Description (_Example_) | diff --git a/src/engines/forgejo/release.rs b/src/engines/forgejo/release.rs index 0e91be8..d863357 100644 --- a/src/engines/forgejo/release.rs +++ b/src/engines/forgejo/release.rs @@ -4,16 +4,23 @@ use std::sync::atomic::{AtomicBool, Ordering}; use tracing_batteries::prelude::*; use crate::{ - BackupEntity, FilterValue, Filterable, - engines::BackupState, - entities::{Credentials, HttpFile}, + engines::{BackupState, summarize_states}, + entities::{Credentials, HttpFile, Release}, errors::HumanizableError, - helpers::forgejo::{CreateReleaseOptions, CreateReleaseResult, ForgejoClient}, + helpers::forgejo::{ + CreateReleaseOptions, CreateReleaseResult, EditReleaseOptions, ForgejoClient, + }, target::RemoteTarget, }; -/// An engine which uploads release artifacts to a Forgejo instance, creating -/// the corresponding release if it does not yet exist. +/// Artifacts which are generated for local backups but should not be uploaded +/// as release assets when replicating to Forgejo. The source tarball is +/// produced by GitHub on demand (and Forgejo generates its own), while the +/// release notes are replicated into the Forgejo release description instead. +const EXCLUDED_ASSETS: &[&str] = &["source.tar.gz", "RELEASE_NOTES.md"]; + +/// An engine which replicates releases (their notes and artifacts) to a Forgejo +/// instance, creating or updating the corresponding release as required. #[derive(Clone, Default)] pub struct ForgejoReleaseEngine { client: ForgejoClient, @@ -24,7 +31,7 @@ impl ForgejoReleaseEngine { #[tracing::instrument(skip(self, entity, target, cancel), fields(entity = %entity))] pub async fn backup( &self, - entity: &HttpFile, + entity: &Release, target: &RemoteTarget, cancel: &AtomicBool, ) -> Result { @@ -32,62 +39,135 @@ impl ForgejoReleaseEngine { return Ok(BackupState::Skipped); } - let (repo, tag, asset_name) = parse_release_path(entity)?; - - let release = match self.client.get_release_by_tag(target, &repo, &tag).await? { - Some(release) => release, - None => { - trace!("Release {tag} does not exist on Forgejo, creating it."); - let draft = matches!(entity.get("release.draft"), FilterValue::Bool(true)); - let prerelease = - matches!(entity.get("release.prerelease"), FilterValue::Bool(true)); - let options = CreateReleaseOptions::new(tag.clone()) - .with_draft(draft) - .with_prerelease(prerelease); - - match self.client.create_release(target, &repo, &options).await? { - CreateReleaseResult::Created(release) => release, - CreateReleaseResult::AlreadyExists => { - // Forgejo reports a 409 Conflict when a release already - // exists for this tag, even though the lookup above - // returned nothing. This happens for draft releases our - // token cannot surface, or for tags synced onto a - // mirrored repository. Try the lookup once more, and if - // the release still cannot be retrieved skip the asset - // rather than failing the entire backup policy. - match self.client.get_release_by_tag(target, &repo, &tag).await? { - Some(release) => release, - None => { - warn!( - "A release for tag '{tag}' already exists on the Forgejo target but could not be retrieved; skipping asset '{asset_name}'." - ); - return Ok(BackupState::Skipped); - } - } - } - } - } + let repo = forgejo_repo_name(&entity.full_name); + + let (release, release_state) = match self.ensure_release(target, &repo, entity).await? { + Some(result) => result, + None => return Ok(BackupState::Skipped), }; - if release.has_asset(&asset_name) { - return Ok(BackupState::Unchanged(Some(format!("asset {asset_name}")))); + let mut states = vec![release_state]; + + for asset in &entity.assets { + if cancel.load(Ordering::Relaxed) { + return Ok(BackupState::Skipped); + } + + let asset_name = asset_file_name(asset); + if EXCLUDED_ASSETS.contains(&asset_name) { + continue; + } + + if release.has_asset(asset_name) { + states.push(BackupState::Unchanged(Some(format!("asset {asset_name}")))); + continue; + } + + let data = self.download_asset(asset).await?; + + if cancel.load(Ordering::Relaxed) { + return Ok(BackupState::Skipped); + } + + self.client + .upload_release_asset(target, &repo, release.id, asset_name, data) + .await?; + + states.push(BackupState::New(Some(format!("asset {asset_name}")))); } - if cancel.load(Ordering::Relaxed) { - return Ok(BackupState::Skipped); + Ok(summarize_states(&states)) + } + + /// Fetches the Forgejo release for the source release's tag, creating it + /// (or updating its notes) as required so that it matches the source. + /// + /// Returns `None` when a release already exists but cannot be retrieved + /// (for example a hidden draft synced onto a mirror), in which case the + /// backup of this release is skipped rather than failing the whole policy. + async fn ensure_release( + &self, + target: &RemoteTarget, + repo: &str, + entity: &Release, + ) -> Result, crate::Error> { + if let Some(release) = self + .client + .get_release_by_tag(target, repo, &entity.tag) + .await? + { + let state = self + .sync_release_notes(target, repo, &release, entity) + .await?; + return Ok(Some((release, state))); + } + + trace!( + "Release {} does not exist on Forgejo, creating it.", + entity.tag + ); + let options = CreateReleaseOptions::new(entity.tag.clone()) + .with_draft(entity.draft) + .with_prerelease(entity.prerelease) + .with_body(entity.body.clone()); + + match self.client.create_release(target, repo, &options).await? { + CreateReleaseResult::Created(release) => Ok(Some(( + release, + BackupState::New(Some("release".to_string())), + ))), + CreateReleaseResult::AlreadyExists => { + // Forgejo reports a 409 Conflict when a release already exists + // for this tag, even though the lookup above returned nothing. + // This happens for draft releases our token cannot surface, or + // for tags synced onto a mirrored repository. Try the lookup + // once more, and if the release still cannot be retrieved skip + // it rather than failing the entire backup policy. + match self + .client + .get_release_by_tag(target, repo, &entity.tag) + .await? + { + Some(release) => { + let state = self + .sync_release_notes(target, repo, &release, entity) + .await?; + Ok(Some((release, state))) + } + None => { + warn!( + "A release for tag '{}' already exists on the Forgejo target but could not be retrieved; skipping it.", + entity.tag + ); + Ok(None) + } + } + } } + } - let data = self.download_asset(entity).await?; + /// Updates the Forgejo release's notes to match the source if they differ. + async fn sync_release_notes( + &self, + target: &RemoteTarget, + repo: &str, + release: &crate::helpers::forgejo::Release, + entity: &Release, + ) -> Result { + let desired = entity.body.as_deref().unwrap_or(""); + let existing = release.body.as_deref().unwrap_or(""); - if cancel.load(Ordering::Relaxed) { - return Ok(BackupState::Skipped); + if desired == existing { + return Ok(BackupState::Unchanged(Some("release".to_string()))); } + trace!("Release notes for {} differ, updating them.", entity.tag); + let options = EditReleaseOptions::new().with_body(entity.body.clone()); self.client - .upload_release_asset(target, &repo, release.id, &asset_name, data) + .update_release(target, repo, release.id, &options) .await?; - Ok(BackupState::New(Some(format!("asset {asset_name}")))) + Ok(BackupState::Updated(Some("release notes".to_string()))) } async fn download_asset(&self, entity: &HttpFile) -> Result, crate::Error> { @@ -131,23 +211,18 @@ impl ForgejoReleaseEngine { } } -/// Release artifacts are named `owner/repo/tag/asset`; we extract the repo, -/// tag, and asset name so we can address the corresponding Forgejo release. -fn parse_release_path(entity: &HttpFile) -> Result<(String, String, String), crate::Error> { - let parts: Vec<&str> = entity.name().splitn(4, '/').collect(); - if parts.len() != 4 { - return Err(human_errors::user( - format!( - "The release artifact '{}' did not have the expected 'owner/repo/tag/asset' structure.", - entity.name() - ), - &["This is likely a bug in github-backup, please report it to us on GitHub."], - )); - } +/// Forgejo repository names cannot contain a `/`, so we use the final path +/// segment of the source repository's name as the Forgejo repository name. +fn forgejo_repo_name(full_name: &str) -> String { + full_name + .rsplit('/') + .next() + .unwrap_or(full_name) + .to_string() +} - Ok(( - parts[1].to_string(), - parts[2].to_string(), - parts[3].to_string(), - )) +/// Release assets are named `owner/repo/tag/asset`; the file name uploaded to +/// Forgejo is the final path segment. +fn asset_file_name(asset: &HttpFile) -> &str { + asset.name.rsplit('/').next().unwrap_or(&asset.name) } diff --git a/src/engines/mod.rs b/src/engines/mod.rs index d0c1cad..c3875de 100644 --- a/src/engines/mod.rs +++ b/src/engines/mod.rs @@ -1,13 +1,15 @@ mod forgejo; mod git; mod http_file; +mod release; pub use forgejo::{ForgejoReleaseEngine, ForgejoRepoEngine}; pub use git::GitEngine; pub use http_file::HttpFileEngine; +pub use release::ReleaseEngine; use crate::BackupEntity; -use crate::entities::{GitRepo, HttpFile}; +use crate::entities::GitRepo; use crate::target::{BackupTarget, RemoteTargetKind}; use std::fmt::Display; use std::sync::atomic::AtomicBool; @@ -67,45 +69,6 @@ impl BackupEngine for RepoEngine { } } -/// A composite engine which backs up release artifacts either to the local -/// filesystem or to a Forgejo instance, depending on the configured target. -#[derive(Clone, Default)] -pub struct ReleaseEngine { - http: HttpFileEngine, - forgejo: ForgejoReleaseEngine, -} - -impl ReleaseEngine { - pub fn new() -> Self { - Self::default() - } -} - -#[async_trait::async_trait] -impl BackupEngine for ReleaseEngine { - async fn backup( - &self, - entity: &HttpFile, - target: &BackupTarget, - cancel: &AtomicBool, - ) -> Result { - match target { - BackupTarget::FileSystem(path) => self.http.backup(entity, path, cancel).await, - BackupTarget::Remote(remote) => match remote.kind { - RemoteTargetKind::ForgejoRelease => { - self.forgejo.backup(entity, remote, cancel).await - } - RemoteTargetKind::ForgejoRepo => Err(human_errors::user( - "You have configured a 'forgejo/repo' target for a release backup, which is not supported.", - &[ - "Use a 'forgejo/release' target to back up release artifacts, or change the policy 'kind' to 'github/repo' to mirror repositories.", - ], - )), - }, - } - } -} - impl Display for BackupState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -119,3 +82,50 @@ impl Display for BackupState { } } } + +/// Combines the backup states of the individual components of a composite +/// entity (such as a release's assets and notes) into a single state. +/// +/// The combined state reflects the "strongest" change applied: a `New` +/// component takes precedence over an `Updated` one, which takes precedence +/// over `Unchanged`. The description summarises how many components fell into +/// each category. An empty set of components is treated as `Skipped`. +pub(crate) fn summarize_states(states: &[BackupState]) -> BackupState { + let mut new = 0; + let mut updated = 0; + let mut unchanged = 0; + let mut skipped = 0; + + for state in states { + match state { + BackupState::New(_) => new += 1, + BackupState::Updated(_) => updated += 1, + BackupState::Unchanged(_) => unchanged += 1, + BackupState::Skipped => skipped += 1, + } + } + + let summary = [ + (new, "new"), + (updated, "updated"), + (unchanged, "unchanged"), + (skipped, "skipped"), + ] + .iter() + .filter(|(count, _)| *count > 0) + .map(|(count, label)| format!("{count} {label}")) + .collect::>() + .join(", "); + + let summary = Some(summary).filter(|s| !s.is_empty()); + + if new > 0 { + BackupState::New(summary) + } else if updated > 0 { + BackupState::Updated(summary) + } else if unchanged > 0 { + BackupState::Unchanged(summary) + } else { + BackupState::Skipped + } +} diff --git a/src/engines/release.rs b/src/engines/release.rs new file mode 100644 index 0000000..e5c0342 --- /dev/null +++ b/src/engines/release.rs @@ -0,0 +1,196 @@ +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; + +use human_errors::ResultExt; + +use crate::{ + BackupEntity, + engines::{BackupState, ForgejoReleaseEngine, HttpFileEngine, summarize_states}, + entities::Release, + target::{BackupTarget, RemoteTargetKind}, +}; + +/// The file name used to store a release's notes when backing up to the local +/// filesystem. +const RELEASE_NOTES_FILE: &str = "RELEASE_NOTES.md"; + +/// A composite engine which backs up releases (their artifacts and notes) +/// either to the local filesystem or to a Forgejo instance, depending on the +/// configured target. +#[derive(Clone, Default)] +pub struct ReleaseEngine { + http: HttpFileEngine, + forgejo: ForgejoReleaseEngine, +} + +impl ReleaseEngine { + pub fn new() -> Self { + Self::default() + } + + /// Backs up a release's assets and notes to the local filesystem. + async fn backup_to_filesystem( + &self, + entity: &Release, + path: &Path, + cancel: &AtomicBool, + ) -> Result { + let mut states = Vec::with_capacity(entity.assets.len() + 1); + + for asset in &entity.assets { + if cancel.load(Ordering::Relaxed) { + return Ok(BackupState::Skipped); + } + + states.push(self.http.backup(asset, path, cancel).await?); + } + + if let Some(state) = self.write_release_notes(entity, path).await? { + states.push(state); + } + + Ok(summarize_states(&states)) + } + + /// Writes the release notes to a `RELEASE_NOTES.md` file alongside the + /// release's assets, returning the resulting backup state (or `None` when + /// the release has no notes to back up). + async fn write_release_notes( + &self, + entity: &Release, + path: &Path, + ) -> Result, crate::Error> { + let body = match &entity.body { + Some(body) if !body.is_empty() => body, + _ => return Ok(None), + }; + + let dir = path.join(entity.target_path()); + let notes_path = dir.join(RELEASE_NOTES_FILE); + + tokio::fs::create_dir_all(&dir).await.wrap_user_err( + format!("Unable to create backup directory '{}'.", dir.display()), + &["Make sure that you have permission to create the directory."], + )?; + + let existing = tokio::fs::read_to_string(¬es_path).await.ok(); + + let state = match existing { + Some(existing) if existing == *body => { + BackupState::Unchanged(Some("release notes".to_string())) + } + Some(_) => BackupState::Updated(Some("release notes".to_string())), + None => BackupState::New(Some("release notes".to_string())), + }; + + if matches!(state, BackupState::Unchanged(_)) { + return Ok(Some(state)); + } + + tokio::fs::write(¬es_path, body).await.wrap_user_err( + format!( + "Unable to write release notes to '{}'.", + notes_path.display() + ), + &["Make sure that you have permission to write to this file/directory and try again."], + )?; + + Ok(Some(state)) + } +} + +#[async_trait::async_trait] +impl crate::engines::BackupEngine for ReleaseEngine { + async fn backup( + &self, + entity: &Release, + target: &BackupTarget, + cancel: &AtomicBool, + ) -> Result { + match target { + BackupTarget::FileSystem(path) => self.backup_to_filesystem(entity, path, cancel).await, + BackupTarget::Remote(remote) => match remote.kind { + RemoteTargetKind::ForgejoRelease => { + self.forgejo.backup(entity, remote, cancel).await + } + RemoteTargetKind::ForgejoRepo => Err(human_errors::user( + "You have configured a 'forgejo/repo' target for a release backup, which is not supported.", + &[ + "Use a 'forgejo/release' target to back up release artifacts, or change the policy 'kind' to 'github/repo' to mirror repositories.", + ], + )), + }, + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::AtomicBool; + + use crate::engines::BackupEngine; + + use super::*; + + static CANCEL: AtomicBool = AtomicBool::new(false); + + fn release_with_notes(body: Option<&str>) -> Release { + Release::new("octocat/example/v1.0", "octocat/example", "v1.0") + .with_body(body.map(|b| b.to_string())) + } + + #[tokio::test] + async fn writes_release_notes_to_filesystem() { + let temp_dir = tempfile::tempdir().expect("a temporary directory"); + let engine = ReleaseEngine::new(); + let target = BackupTarget::FileSystem(temp_dir.path().to_path_buf()); + + let entity = release_with_notes(Some("These are the release notes.")); + + // First run creates the notes file. + let state = engine.backup(&entity, &target, &CANCEL).await.unwrap(); + assert!(matches!(state, BackupState::New(_))); + + let notes_path = temp_dir + .path() + .join("octocat/example/v1.0") + .join(RELEASE_NOTES_FILE); + assert_eq!( + std::fs::read_to_string(¬es_path).unwrap(), + "These are the release notes." + ); + + // Re-running with the same notes leaves them unchanged. + let state = engine.backup(&entity, &target, &CANCEL).await.unwrap(); + assert!(matches!(state, BackupState::Unchanged(_))); + + // Changing the notes results in an update. + let updated = release_with_notes(Some("Revised release notes.")); + let state = engine.backup(&updated, &target, &CANCEL).await.unwrap(); + assert!(matches!(state, BackupState::Updated(_))); + assert_eq!( + std::fs::read_to_string(¬es_path).unwrap(), + "Revised release notes." + ); + } + + #[tokio::test] + async fn skips_filesystem_backup_without_notes_or_assets() { + let temp_dir = tempfile::tempdir().expect("a temporary directory"); + let engine = ReleaseEngine::new(); + let target = BackupTarget::FileSystem(temp_dir.path().to_path_buf()); + + let entity = release_with_notes(None); + + let state = engine.backup(&entity, &target, &CANCEL).await.unwrap(); + assert!(matches!(state, BackupState::Skipped)); + + assert!( + !temp_dir + .path() + .join("octocat/example/v1.0") + .join(RELEASE_NOTES_FILE) + .exists() + ); + } +} diff --git a/src/entities/mod.rs b/src/entities/mod.rs index d66249d..806ba7e 100644 --- a/src/entities/mod.rs +++ b/src/entities/mod.rs @@ -1,10 +1,12 @@ mod credentials; #[macro_use] mod macros; +mod release; use crate::{FilterValue, Filterable}; pub use credentials::Credentials; +pub use release::Release; use std::collections::HashMap; use unicase::UniCase; diff --git a/src/entities/release.rs b/src/entities/release.rs new file mode 100644 index 0000000..f1eec1e --- /dev/null +++ b/src/entities/release.rs @@ -0,0 +1,17 @@ +use crate::{FilterValue, entities::HttpFile}; + +entity!(Release(full_name: F => String, tag: T => String) { + with_body => body: Option, + with_draft => draft: bool, + with_prerelease => prerelease: bool, + with_assets => assets: Vec, +}); + +#[allow(dead_code)] +impl Release { + /// Adds a single downloadable artifact to the release. + pub fn with_asset(mut self, asset: HttpFile) -> Self { + self.assets.push(asset); + self + } +} diff --git a/src/helpers/forgejo/client.rs b/src/helpers/forgejo/client.rs index c785c90..2531805 100644 --- a/src/helpers/forgejo/client.rs +++ b/src/helpers/forgejo/client.rs @@ -9,7 +9,8 @@ use crate::{ }; use super::entities::{ - CreateReleaseOptions, CreateReleaseResult, MigrateRepoOptions, Release, Repository, + CreateReleaseOptions, CreateReleaseResult, EditReleaseOptions, MigrateRepoOptions, Release, + Repository, }; /// A thin client for the subset of the Forgejo REST API that we need in order @@ -127,6 +128,28 @@ impl ForgejoClient { )) } + /// Updates an existing release on the Forgejo instance, for example to keep + /// its release notes in sync with the source. + pub async fn update_release( + &self, + target: &RemoteTarget, + repo: &str, + release_id: u64, + options: &EditReleaseOptions, + ) -> Result { + let url = target.api_url(&format!( + "repos/{}/{}/releases/{}", + target.owner, repo, release_id + )); + let resp = self + .call(Method::PATCH, &url, &target.credentials, |r| { + r.json(options) + }) + .await?; + let resp = self.ensure_success(resp, "updating a release").await?; + self.parse_json(resp, &url).await + } + /// Uploads an asset to an existing release. pub async fn upload_release_asset( &self, diff --git a/src/helpers/forgejo/entities.rs b/src/helpers/forgejo/entities.rs index b77d39a..ffaee80 100644 --- a/src/helpers/forgejo/entities.rs +++ b/src/helpers/forgejo/entities.rs @@ -104,6 +104,35 @@ impl CreateReleaseOptions { self.prerelease = prerelease; self } + + pub fn with_body(mut self, body: Option) -> Self { + self.body = body; + self + } +} + +/// Options used when editing an existing release on a Forgejo instance. +/// +/// See the Forgejo `PATCH /repos/{owner}/{repo}/releases/{id}` endpoint. +#[derive(Debug, Clone, Default, Serialize)] +pub struct EditReleaseOptions { + #[serde(skip_serializing_if = "Option::is_none")] + pub body: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub prerelease: Option, +} + +impl EditReleaseOptions { + pub fn new() -> Self { + Self::default() + } + + pub fn with_body(mut self, body: Option) -> Self { + self.body = body; + self + } } /// A subset of the fields returned by Forgejo when describing a release. @@ -113,6 +142,8 @@ pub struct Release { #[allow(dead_code)] pub tag_name: String, #[serde(default)] + pub body: Option, + #[serde(default)] pub assets: Vec, } diff --git a/src/helpers/forgejo/mod.rs b/src/helpers/forgejo/mod.rs index e53cbee..5b75a77 100644 --- a/src/helpers/forgejo/mod.rs +++ b/src/helpers/forgejo/mod.rs @@ -2,7 +2,9 @@ mod client; mod entities; pub use client::ForgejoClient; -pub use entities::{CreateReleaseOptions, CreateReleaseResult, MigrateRepoOptions}; +pub use entities::{ + CreateReleaseOptions, CreateReleaseResult, EditReleaseOptions, MigrateRepoOptions, Release, +}; #[cfg(test)] mod tests { @@ -148,4 +150,23 @@ mod tests { .await .unwrap(); } + + #[tokio::test] + async fn update_release() { + let client = + ForgejoClient::default().mock("/api/v1/repos/backups/example/releases/9", |b| { + b.with_body( + r#"{"id": 9, "tag_name": "v2.0", "body": "Updated notes", "assets": []}"#, + ) + }); + + let options = EditReleaseOptions::new().with_body(Some("Updated notes".to_string())); + let release = client + .update_release(&target(), "example", 9, &options) + .await + .unwrap(); + + assert_eq!(release.id, 9); + assert_eq!(release.body.as_deref(), Some("Updated notes")); + } } diff --git a/src/pairing.rs b/src/pairing.rs index e7838b5..bb92723 100644 --- a/src/pairing.rs +++ b/src/pairing.rs @@ -107,15 +107,21 @@ impl< continue; } - match policy.filter.matches(&entity) { - Ok(true) => {}, - Ok(false) => { - yield Ok((entity, BackupState::Skipped)); - continue; - }, - Err(e) => { - yield Err(e); - continue; + // Some sources (such as releases) bundle several filterable + // items into a single entity and apply the policy filter to each + // item themselves. For those we skip the entity-level filtering + // so that filtering keeps operating at the per-item granularity. + if !self.source.filters_internally() { + match policy.filter.matches(&entity) { + Ok(true) => {}, + Ok(false) => { + yield Ok((entity, BackupState::Skipped)); + continue; + }, + Err(e) => { + yield Err(e); + continue; + } } } diff --git a/src/sources/github_releases.rs b/src/sources/github_releases.rs index 72471c8..59129f7 100644 --- a/src/sources/github_releases.rs +++ b/src/sources/github_releases.rs @@ -4,7 +4,7 @@ use tokio_stream::Stream; use crate::{ BackupSource, - entities::{Credentials, HttpFile}, + entities::{Credentials, HttpFile, Release}, helpers::{ GitHubClient, github::{GitHubArtifactKind, GitHubRelease, GitHubRepo, GitHubRepoSourceKind}, @@ -30,7 +30,7 @@ impl GitHubReleasesSource { policy: &'a BackupPolicy, repo: &'a GitHubRepo, cancel: &'a AtomicBool, - ) -> impl Stream> + 'a { + ) -> impl Stream> + 'a { async_stream::stream! { if !repo.has_downloads { return; @@ -46,8 +46,20 @@ impl GitHubReleasesSource { let release: GitHubRelease = release.unwrap(); + let mut entity = Release::new( + format!("{}/{}", &repo.full_name, &release.tag_name), + repo.full_name.as_str(), + release.tag_name.as_str(), + ) + .with_body(release.body.clone()) + .with_draft(release.draft) + .with_prerelease(release.prerelease) + .with_metadata_source(repo) + .with_metadata_source(&release); + if let Some(tarball_url) = &release.tarball_url { - yield Ok(HttpFile::new(format!("{}/{}/source.tar.gz", &repo.full_name, &release.tag_name), tarball_url) + entity = entity.with_asset( + HttpFile::new(format!("{}/{}/source.tar.gz", &repo.full_name, &release.tag_name), tarball_url) .with_metadata_source(repo) .with_metadata_source(&release) .with_metadata("asset.source-code", true) @@ -72,7 +84,8 @@ impl GitHubReleasesSource { let asset_url = format!("{}/releases/assets/{}", repo.url, asset.id); - yield Ok(HttpFile::new(format!("{}/{}/{}", &repo.full_name, &release.tag_name, &asset.name), asset_url) + entity = entity.with_asset( + HttpFile::new(format!("{}/{}/{}", &repo.full_name, &release.tag_name, &asset.name), asset_url) .with_content_type(Some("application/octet-stream".to_string())) .with_credentials(match &policy.credentials { Credentials::Token(token) => Credentials::UsernamePassword { @@ -86,16 +99,54 @@ impl GitHubReleasesSource { .with_metadata_source(&release) .with_metadata_source(asset)); } + + // Apply the policy filter at the granularity of individual release + // assets, preserving the ability to control backups using the + // `asset.*`, `release.*` and `repo.*` filter fields. + let mut assets = Vec::with_capacity(entity.assets.len()); + for asset in std::mem::take(&mut entity.assets) { + match policy.filter.matches(&asset) { + Ok(true) => assets.push(asset), + Ok(false) => {}, + Err(e) => { + yield Err(e); + } + } + } + entity.assets = assets; + + // The release notes are governed by the release-level fields + // (`repo.*` / `release.*`); drop them when the release is filtered + // out at that level. + match policy.filter.matches(&entity) { + Ok(true) => {}, + Ok(false) => { entity.body = None; }, + Err(e) => { + yield Err(e); + continue; + } + } + + // Skip releases which have nothing left to back up after filtering. + if entity.assets.is_empty() && entity.body.as_deref().is_none_or(str::is_empty) { + continue; + } + + yield Ok(entity); } } } } -impl BackupSource for GitHubReleasesSource { +impl BackupSource for GitHubReleasesSource { fn kind(&self) -> &str { GitHubArtifactKind::Release.as_str() } + fn filters_internally(&self) -> bool { + true + } + fn validate(&self, policy: &BackupPolicy) -> Result<(), human_errors::Error> { let target: GitHubRepoSourceKind = policy.from.as_str().parse()?; @@ -118,7 +169,7 @@ impl BackupSource for GitHubReleasesSource { &'a self, policy: &'a BackupPolicy, cancel: &'a AtomicBool, - ) -> impl Stream> + 'a { + ) -> impl Stream> + 'a { let target: GitHubRepoSourceKind = policy.from.as_str().parse().unwrap(); let url = format!( "{}/{}?{}", @@ -236,7 +287,7 @@ mod tests { } #[rstest] - #[case("github.releases.0.json", 93)] + #[case("github.releases.0.json", 31)] #[tokio::test] async fn get_releases_mocked(#[case] filename: &str, #[case] expected_entries: usize) { use tokio_stream::StreamExt; @@ -265,7 +316,14 @@ mod tests { let mut count = 0; while let Some(release) = stream.next().await { - println!("{}", release.expect("Failed to load release")); + let release = release.expect("Failed to load release"); + println!("{}", release); + + // Each release in the fixture bundles a tarball plus two uploaded + // assets and includes a set of release notes. + assert_eq!(release.assets.len(), 3); + assert!(release.body.is_some()); + count += 1; } @@ -275,4 +333,68 @@ mod tests { expected_entries, count ); } + + #[rstest] + // Only the matching asset is retained, and the release notes are dropped + // because the release does not match at the `release.*` / `repo.*` level. + #[case("asset.name == \"client.exe\"", 31, 1, false)] + // A `release.*` filter keeps every asset and the release notes. + #[case("release.prerelease == false", 31, 3, true)] + // A filter which matches no assets skips the release entirely. + #[case("asset.name == \"does-not-exist\"", 0, 0, false)] + #[tokio::test] + async fn get_releases_filtered( + #[case] filter: &str, + #[case] expected_releases: usize, + #[case] expected_assets: usize, + #[case] expect_notes: bool, + ) { + use tokio_stream::StreamExt; + + let source = GitHubReleasesSource::with_client( + GitHubClient::default() + .mock("/users/octocat/repos", |b| { + b.with_body_from_file("github.repos.0.json") + }) + .mock("/repos/octocat/repo/releases", |b| { + b.with_body_from_file("github.releases.0.json") + }), + ); + + let policy: BackupPolicy = serde_yaml::from_str(&format!( + r#" + kind: github/release + from: users/octocat + to: /tmp + filter: '{filter}' + "# + )) + .unwrap(); + + let stream = source.load(&policy, &CANCEL); + tokio::pin!(stream); + + let mut count = 0; + while let Some(release) = stream.next().await { + let release = release.expect("Failed to load release"); + assert_eq!( + release.assets.len(), + expected_assets, + "unexpected asset count for filter '{filter}'" + ); + assert_eq!( + release.body.is_some(), + expect_notes, + "unexpected release notes presence for filter '{filter}'" + ); + + count += 1; + } + + assert_eq!( + count, expected_releases, + "Expected {} releases for filter '{}', got {}", + expected_releases, filter, count + ); + } } diff --git a/src/sources/mod.rs b/src/sources/mod.rs index f921f47..e0d10aa 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -18,4 +18,16 @@ pub trait BackupSource { policy: &'a BackupPolicy, cancel: &'a AtomicBool, ) -> impl Stream> + 'a; + + /// Indicates whether this source applies the policy's filter itself (for + /// example, per child artifact) rather than relying on the entity-level + /// filtering performed by the [`crate::pairing::Pairing`]. + /// + /// Sources which bundle multiple filterable items into a single entity + /// (such as a release and its assets) should return `true` and apply the + /// filter to each item during [`BackupSource::load`], so that filtering + /// continues to operate at the granularity of the individual items. + fn filters_internally(&self) -> bool { + false + } }