From 3b3a8f09cef2927479ea02e9b9ab25e339a226a7 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Mon, 11 May 2026 12:54:56 +0200 Subject: [PATCH] feat: Fetch tags from GitHub, use only latest from SDP release line --- Makefile | 2 +- README.md | 25 +++--- docs-generator/doc/doc.go | 6 +- docs-generator/gitter/gitter.go | 6 +- docs-generator/pkg/config/config.go | 126 +++++++++++++++++++++++++++- 5 files changed, 143 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 218ad88..82ce3c9 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Set the shell to bash always -SHELL := /bin/bash +SHELL := /usr/bin/env bash # Note: CGO_ENABLED is required for the SQLite3 module. export CGO_ENABLED=1 diff --git a/README.md b/README.md index 7e532a1..998795b 100644 --- a/README.md +++ b/README.md @@ -27,23 +27,18 @@ It stores these in an SQLite database. ## Generating docs Have a look at https://github.com/stackabletech/crddocs for sample usage. -To generate docs you need a yaml configuration file specifying which repos and tags to document. -It should look like this: +To generate docs you need a yaml configuration file listing the repos to document: repos: - airflow-operator: - - "24.7.0" - - "nightly" - druid-operator: - - "24.7.0" - - "nightly" - hbase-operator: - - "24.7.0" - - "nightly" - - platformVersions: - - "24.7.0" - - "nightly" + - airflow-operator + - druid-operator + - hbase-operator + +Tags are auto-discovered: all calver `YY.M.P` tags on each repo at +`github.com/stackabletech/` are listed, only the latest patch per +`YY.M` release line is kept, and `nightly` (tracking `main`) is always added. +The platform-version list shown on the landing page is the union across all +repos. You also need a HTML file template and a directory of static files. diff --git a/docs-generator/doc/doc.go b/docs-generator/doc/doc.go index dc47725..ff60b91 100644 --- a/docs-generator/doc/doc.go +++ b/docs-generator/doc/doc.go @@ -157,6 +157,10 @@ func main() { log.Fatalf("Error loading config: %s: %v", configFile, err) panic(err) } + if err := conf.ResolveTags(); err != nil { + log.Fatalf("Error resolving tags: %v", err) + panic(err) + } // generate landing page(s) home(db, outDir, "", conf.PlatformVersions) @@ -165,7 +169,7 @@ func main() { } // generate doc pages for all repos and CRDs - for repo, tags := range conf.Repos { + for repo, tags := range conf.Tags { org(db, outDir, repo, "") for _, tag := range tags { org(db, outDir, repo, tag) diff --git a/docs-generator/gitter/gitter.go b/docs-generator/gitter/gitter.go index 315dafa..455e07f 100644 --- a/docs-generator/gitter/gitter.go +++ b/docs-generator/gitter/gitter.go @@ -72,9 +72,13 @@ func main() { log.Fatalf("Error loading config: %s: %v", configFile, err) panic(err) } + if err := conf.ResolveTags(); err != nil { + log.Fatalf("Error resolving tags: %v", err) + panic(err) + } // index repos - for repo, tags := range conf.Repos { + for repo, tags := range conf.Tags { log.Printf("Indexing repo %s ...\n", repo) for _, tag := range tags { log.Printf("... at tag: %s ...\n", tag) diff --git a/docs-generator/pkg/config/config.go b/docs-generator/pkg/config/config.go index 9670ea4..095991c 100644 --- a/docs-generator/pkg/config/config.go +++ b/docs-generator/pkg/config/config.go @@ -1,15 +1,28 @@ package config import ( + "fmt" "log" "os" + "regexp" + "sort" + "strconv" + "strings" + "github.com/go-git/go-git/v5" + gitconfig "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/storage/memory" "gopkg.in/yaml.v2" ) +const NightlyTag = "nightly" + type Config struct { - Repos map[string][]string `yaml:"repos"` - PlatformVersions []string `yaml:"platformVersions"` + Repos []string `yaml:"repos"` + + // Tags and PlatformVersions are populated by ResolveTags. + Tags map[string][]string `yaml:"-"` + PlatformVersions []string `yaml:"-"` } func (c *Config) NewConfigFromFile(filePath string) error { @@ -19,11 +32,116 @@ func (c *Config) NewConfigFromFile(filePath string) error { return err } - err = yaml.Unmarshal(yamlFile, c) - if err != nil { + if err := yaml.Unmarshal(yamlFile, c); err != nil { log.Fatalf("Error unmarshalling YAML: %v", err) return err } return nil } + +// ResolveTags lists remote tags for each configured repo, keeps only the +// latest patch per YY.M release line, appends `nightly`, and fills +// c.Tags and c.PlatformVersions. PlatformVersions is the union across all +// repos, newest calver first, with nightly last (matching the previous +// hand-maintained order). +func (c *Config) ResolveTags() error { + c.Tags = map[string][]string{} + platformSet := map[string]struct{}{} + + for _, repo := range c.Repos { + url := "https://github.com/stackabletech/" + repo + log.Printf("Listing tags for %s ...", repo) + tags, err := listLatestPatches(url) + if err != nil { + return fmt.Errorf("listing tags for %s: %w", repo, err) + } + c.Tags[repo] = append(tags, NightlyTag) + + for _, t := range tags { + platformSet[t] = struct{}{} + } + } + + platform := make([]string, 0, len(platformSet)) + for v := range platformSet { + platform = append(platform, v) + } + sortCalverDesc(platform) + c.PlatformVersions = append(platform, NightlyTag) + + return nil +} + +type calver struct{ y, m, p int } + +var calverRe = regexp.MustCompile(`^(\d{2})\.(\d{1,2})\.(\d+)$`) + +func parseCalver(s string) (calver, bool) { + m := calverRe.FindStringSubmatch(s) + if m == nil { + return calver{}, false + } + y, _ := strconv.Atoi(m[1]) + mo, _ := strconv.Atoi(m[2]) + p, _ := strconv.Atoi(m[3]) + return calver{y, mo, p}, true +} + +func calverLess(a, b calver) bool { + if a.y != b.y { + return a.y < b.y + } + if a.m != b.m { + return a.m < b.m + } + return a.p < b.p +} + +func sortCalverDesc(versions []string) { + sort.Slice(versions, func(i, j int) bool { + a, _ := parseCalver(versions[i]) + b, _ := parseCalver(versions[j]) + return calverLess(b, a) + }) +} + +// listLatestPatches returns the latest-patch calver tag per YY.M release +// line for the given remote, newest first. +func listLatestPatches(url string) ([]string, error) { + rem := git.NewRemote(memory.NewStorage(), &gitconfig.RemoteConfig{ + Name: "origin", + URLs: []string{url}, + }) + refs, err := rem.List(&git.ListOptions{}) + if err != nil { + return nil, err + } + + type line struct{ y, m int } + latest := map[line]calver{} + names := map[line]string{} + + for _, ref := range refs { + if !ref.Name().IsTag() { + continue + } + name := strings.TrimPrefix(ref.Name().String(), "refs/tags/") + cv, ok := parseCalver(name) + if !ok { + continue + } + key := line{cv.y, cv.m} + if cur, exists := latest[key]; !exists || calverLess(cur, cv) { + latest[key] = cv + names[key] = name + } + } + + out := make([]string, 0, len(names)) + for _, n := range names { + out = append(out, n) + } + sortCalverDesc(out) + return out, nil +}