From 7dff746fa6394feb714a50fe7fbe5f38e8f2ddb3 Mon Sep 17 00:00:00 2001 From: Alexis-Maurer Fortin Date: Tue, 7 Apr 2026 16:48:35 -0400 Subject: [PATCH 1/5] added the observer interface for monitoring analysis progress. converted progress bar to it. --- analyze/analyze.go | 44 +++++++++--- analyze/analyze_test.go | 118 +++++++++++++++++++++++++++++++ analyze/observer.go | 68 ++++++++++++++++++ analyze/progress_bar_observer.go | 66 +++++++++++++++++ cmd/root.go | 2 + 5 files changed, 289 insertions(+), 9 deletions(-) create mode 100644 analyze/observer.go create mode 100644 analyze/progress_bar_observer.go diff --git a/analyze/analyze.go b/analyze/analyze.go index 865183a1..46e50830 100644 --- a/analyze/analyze.go +++ b/analyze/analyze.go @@ -96,6 +96,14 @@ type Analyzer struct { Formatter Formatter Config *models.Config Opa *opa.Opa + Observer ProgressObserver +} + +func (a *Analyzer) observer() ProgressObserver { + if a.Observer != nil { + return a.Observer + } + return noopObserver{} } func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutines *int) ([]*models.PackageInsights, error) { @@ -115,7 +123,7 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine inventory := scanner.NewInventory(a.Opa, pkgsupplyClient, provider, providerVersion) log.Debug().Msgf("Starting repository analysis for organization: %s on %s", org, provider) - bar := a.ProgressBar(0, "Analyzing repositories") + obs := a.observer() var reposWg sync.WaitGroup errChan := make(chan error, 1) @@ -143,17 +151,16 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine continue } if repoBatch.TotalCount != 0 { - bar.ChangeMax(repoBatch.TotalCount) + obs.OnDiscoveryCompleted(org, repoBatch.TotalCount) } for _, repo := range repoBatch.Repositories { if a.Config.IgnoreForks && repo.GetIsFork() { - bar.ChangeMax(repoBatch.TotalCount - 1) + obs.OnRepoSkipped(repo.GetRepoIdentifier(), "fork") continue } if repo.GetSize() == 0 { - bar.ChangeMax(repoBatch.TotalCount - 1) - log.Info().Str("repo", repo.GetRepoIdentifier()).Msg("Skipping empty repository") + obs.OnRepoSkipped(repo.GetRepoIdentifier(), "empty") continue } if err := goRoutineLimitSem.Acquire(ctx, 1); err != nil { @@ -166,9 +173,11 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine defer goRoutineLimitSem.Release(1) defer reposWg.Done() repoNameWithOwner := repo.GetRepoIdentifier() + obs.OnRepoStarted(repoNameWithOwner) repoKey, err := a.cloneRepo(ctx, repo.BuildGitURL(a.ScmClient.GetProviderBaseURL()), a.ScmClient.GetToken(), "HEAD") if err != nil { log.Error().Err(err).Str("repo", repoNameWithOwner).Msg("failed to clone repo") + obs.OnRepoError(repoNameWithOwner, err) return } defer a.GitClient.Cleanup(repoKey) @@ -176,12 +185,14 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine pkg, err := a.GeneratePackageInsights(ctx, repoKey, repo, "HEAD") if err != nil { log.Error().Err(err).Str("repo", repoNameWithOwner).Msg("failed to generate package insights") + obs.OnRepoError(repoNameWithOwner, err) return } files, err := a.GitClient.ListFiles(repoKey, []string{".yml", ".yaml"}) if err != nil { log.Error().Err(err).Str("repo", repoNameWithOwner).Msg("failed to list files") + obs.OnRepoError(repoNameWithOwner, err) return } @@ -199,6 +210,7 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine scannedPkg, err := inventory.ScanPackageScanner(ctx, *pkg, memScanner) if err != nil { log.Error().Err(err).Str("repo", repoNameWithOwner).Msg("failed to scan package") + obs.OnRepoError(repoNameWithOwner, err) return } @@ -208,7 +220,7 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine log.Error().Msg("Context canceled while sending package to channel") return } - _ = bar.Add(1) + obs.OnRepoCompleted(repoNameWithOwner, scannedPkg) }(repo) } } @@ -227,12 +239,13 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine } } - _ = bar.Finish() - + obs.OnFinalizeStarted(len(scannedPackages)) err = a.finalizeAnalysis(ctx, scannedPackages) if err != nil { + obs.OnFinalizeCompleted() return scannedPackages, err } + obs.OnFinalizeCompleted() return scannedPackages, nil } @@ -259,10 +272,12 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, inventory := scanner.NewInventory(a.Opa, pkgsupplyClient, provider, providerVersion) + obs := a.observer() log.Debug().Msgf("Starting repository analysis for: %s/%s on %s", org, repoName, provider) bar := a.ProgressBar(3, "Cloning repository") _ = bar.RenderBlank() + obs.OnRepoStarted(repoString) repoUrl := repo.BuildGitURL(a.ScmClient.GetProviderBaseURL()) repoKey, err := a.fetchCone(ctx, repoUrl, a.ScmClient.GetToken(), "refs/heads/*:refs/remotes/origin/*", ".github/workflows") if err != nil { @@ -363,6 +378,9 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, } _ = bar.Finish() + obs.OnRepoCompleted(repoString, scannedPackage) + + obs.OnFinalizeStarted(1) if *expand { expanded := []results.Finding{} for _, finding := range scannedPackage.FindingsResults.Findings { @@ -385,6 +403,7 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, scannedPackage.FindingsResults.Findings = expanded if err := a.Formatter.Format(ctx, []*models.PackageInsights{scannedPackage}); err != nil { + obs.OnFinalizeCompleted() return nil, fmt.Errorf("failed to finalize analysis of package: %w", err) } } else { @@ -398,10 +417,11 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, } if err := a.Formatter.FormatWithPath(ctx, []*models.PackageInsights{scannedPackage}, results); err != nil { + obs.OnFinalizeCompleted() return nil, fmt.Errorf("failed to finalize analysis of package: %w", err) } - } + obs.OnFinalizeCompleted() return scannedPackage, nil } @@ -427,10 +447,12 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin pkgsupplyClient := pkgsupply.NewStaticClient() inventory := scanner.NewInventory(a.Opa, pkgsupplyClient, provider, providerVersion) + obs := a.observer() log.Debug().Msgf("Starting repository analysis for: %s/%s on %s", org, repoName, provider) bar := a.ProgressBar(2, "Cloning repository") _ = bar.RenderBlank() + obs.OnRepoStarted(repoString) repoKey, err := a.cloneRepo(ctx, repo.BuildGitURL(a.ScmClient.GetProviderBaseURL()), a.ScmClient.GetToken(), ref) if err != nil { return nil, err @@ -466,11 +488,15 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin return nil, err } _ = bar.Finish() + obs.OnRepoCompleted(repoString, scannedPackage) + obs.OnFinalizeStarted(1) err = a.finalizeAnalysis(ctx, []*models.PackageInsights{scannedPackage}) if err != nil { + obs.OnFinalizeCompleted() return nil, err } + obs.OnFinalizeCompleted() return scannedPackage, nil } diff --git a/analyze/analyze_test.go b/analyze/analyze_test.go index 3fe4dc6f..d35f84cf 100644 --- a/analyze/analyze_test.go +++ b/analyze/analyze_test.go @@ -2,8 +2,10 @@ package analyze import ( "context" + "errors" "fmt" "strings" + "sync" "testing" "github.com/boostsecurityio/poutine/formatters/noop" @@ -188,3 +190,119 @@ jobs: require.NotNil(t, result) }) } + +// mockObserver records observer events for testing. +type mockObserver struct { + mu sync.Mutex + events []string +} + +func (m *mockObserver) OnDiscoveryCompleted(org string, totalCount int) { + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, fmt.Sprintf("discovery_completed:%s:%d", org, totalCount)) +} +func (m *mockObserver) OnRepoStarted(repo string) { + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, "repo_started:"+repo) +} +func (m *mockObserver) OnRepoCompleted(repo string, _ *models.PackageInsights) { + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, "repo_completed:"+repo) +} +func (m *mockObserver) OnRepoError(repo string, _ error) { + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, "repo_error:"+repo) +} +func (m *mockObserver) OnRepoSkipped(repo string, reason string) { + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, fmt.Sprintf("repo_skipped:%s:%s", repo, reason)) +} +func (m *mockObserver) OnFinalizeStarted(total int) { + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, fmt.Sprintf("finalize_started:%d", total)) +} +func (m *mockObserver) OnFinalizeCompleted() { + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, "finalize_completed") +} + +func TestProgressObserverNilSafe(t *testing.T) { + ctx := context.Background() + opaClient, err := newTestOpa(ctx) + require.NoError(t, err) + + // Observer is nil — should not panic + analyzer := NewAnalyzer(nil, nil, &noop.Format{}, models.DefaultConfig(), opaClient) + assert.Nil(t, analyzer.Observer) + + // AnalyzeManifest doesn't use observer, but this ensures nil Observer doesn't crash + _, err = analyzer.AnalyzeManifest(ctx, strings.NewReader("on: push\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4"), "github-actions") + require.NoError(t, err) +} + +func TestProgressObserverInterface(t *testing.T) { + obs := &mockObserver{} + + // Verify the interface is implemented + var _ ProgressObserver = obs + var _ ProgressObserver = &noopObserver{} + var _ ProgressObserver = &ProgressBarObserver{} + + // Test mock records events + obs.OnDiscoveryCompleted("test-org", 10) + obs.OnRepoStarted("test-org/repo1") + obs.OnRepoCompleted("test-org/repo1", nil) + obs.OnRepoSkipped("test-org/repo2", "fork") + obs.OnRepoError("test-org/repo3", errors.New("clone failed")) + obs.OnFinalizeStarted(1) + obs.OnFinalizeCompleted() + + assert.Equal(t, []string{ + "discovery_completed:test-org:10", + "repo_started:test-org/repo1", + "repo_completed:test-org/repo1", + "repo_skipped:test-org/repo2:fork", + "repo_error:test-org/repo3", + "finalize_started:1", + "finalize_completed", + }, obs.events) +} + +func TestProgressObserverConcurrency(t *testing.T) { + // Exercise the concurrent methods on both observer implementations + // to verify no data races. Run with: go test -race + observers := []ProgressObserver{ + &mockObserver{}, + NewProgressBarObserver(true), + } + + for _, obs := range observers { + obs.OnDiscoveryCompleted("org", 100) + + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + repo := fmt.Sprintf("org/repo-%d", n) + obs.OnRepoStarted(repo) + if n%5 == 0 { + obs.OnRepoError(repo, errors.New("error")) + } else { + obs.OnRepoCompleted(repo, nil) + } + }(i) + } + wg.Wait() + + obs.OnFinalizeStarted(40) + obs.OnFinalizeCompleted() + } +} diff --git a/analyze/observer.go b/analyze/observer.go new file mode 100644 index 00000000..ed3d4df0 --- /dev/null +++ b/analyze/observer.go @@ -0,0 +1,68 @@ +package analyze + +import "github.com/boostsecurityio/poutine/models" + +// ProgressObserver receives structured progress events during analysis. +// All methods must be non-blocking — implementations should not perform +// expensive I/O or hold locks for extended periods. +// +// # Concurrency +// +// During AnalyzeOrg, methods are called from two contexts: +// +// Main goroutine (sequential, never concurrent with each other): +// - OnDiscoveryCompleted +// - OnRepoSkipped +// - OnFinalizeStarted +// - OnFinalizeCompleted +// +// Worker goroutines (called concurrently from multiple goroutines): +// - OnRepoStarted +// - OnRepoCompleted +// - OnRepoError +// +// Implementations MUST ensure that the worker-goroutine methods are +// goroutine-safe. Note that worker methods may also run concurrently +// with the main-goroutine methods above. +// +// During AnalyzeRepo and AnalyzeStaleBranches all methods are called +// from a single goroutine. +type ProgressObserver interface { + // OnDiscoveryCompleted is called from the main goroutine when the + // total repo count is known (first batch with TotalCount > 0). + OnDiscoveryCompleted(org string, totalCount int) + + // OnRepoStarted is called from a worker goroutine when analysis + // of a repo begins. Must be goroutine-safe. + OnRepoStarted(repo string) + + // OnRepoCompleted is called from a worker goroutine when a repo + // finishes successfully. Must be goroutine-safe. + OnRepoCompleted(repo string, pkg *models.PackageInsights) + + // OnRepoError is called from a worker goroutine when a repo + // analysis fails (non-fatal). Must be goroutine-safe. + OnRepoError(repo string, err error) + + // OnRepoSkipped is called from the main goroutine when a repo is + // skipped (fork, empty, etc.). + OnRepoSkipped(repo string, reason string) + + // OnFinalizeStarted is called from the main goroutine when the + // formatting/output phase begins, after all repos are processed. + OnFinalizeStarted(totalPackages int) + + // OnFinalizeCompleted is called from the main goroutine when + // formatting is done. + OnFinalizeCompleted() +} + +type noopObserver struct{} + +func (noopObserver) OnDiscoveryCompleted(string, int) {} +func (noopObserver) OnRepoStarted(string) {} +func (noopObserver) OnRepoCompleted(string, *models.PackageInsights) {} +func (noopObserver) OnRepoError(string, error) {} +func (noopObserver) OnRepoSkipped(string, string) {} +func (noopObserver) OnFinalizeStarted(int) {} +func (noopObserver) OnFinalizeCompleted() {} diff --git a/analyze/progress_bar_observer.go b/analyze/progress_bar_observer.go new file mode 100644 index 00000000..b46e0c2d --- /dev/null +++ b/analyze/progress_bar_observer.go @@ -0,0 +1,66 @@ +package analyze + +import ( + "os" + + "github.com/boostsecurityio/poutine/models" + "github.com/schollz/progressbar/v3" +) + +// ProgressBarObserver implements ProgressObserver by rendering a CLI progress bar. +type ProgressBarObserver struct { + bar *progressbar.ProgressBar + quiet bool +} + +func NewProgressBarObserver(quiet bool) *ProgressBarObserver { + var bar *progressbar.ProgressBar + if quiet { + bar = progressbar.DefaultSilent(0, "Analyzing repositories") + } else { + bar = progressbar.NewOptions64(0, + progressbar.OptionSetDescription("Analyzing repositories"), + progressbar.OptionShowCount(), + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionClearOnFinish(), + ) + } + return &ProgressBarObserver{bar: bar, quiet: quiet} +} + +func (o *ProgressBarObserver) OnDiscoveryCompleted(_ string, totalCount int) { + o.bar.ChangeMax(totalCount) +} + +func (o *ProgressBarObserver) OnRepoStarted(_ string) {} + +func (o *ProgressBarObserver) OnRepoCompleted(_ string, _ *models.PackageInsights) { + _ = o.bar.Add(1) +} + +func (o *ProgressBarObserver) OnRepoError(_ string, _ error) { + _ = o.bar.Add(1) +} + +func (o *ProgressBarObserver) OnRepoSkipped(_ string, _ string) { + o.bar.ChangeMax(o.bar.GetMax() - 1) +} + +func (o *ProgressBarObserver) OnFinalizeStarted(_ int) { + _ = o.bar.Finish() +} + +func (o *ProgressBarObserver) OnFinalizeCompleted() {} + +// ProgressBarForSteps creates a step-counting progress bar for single-repo operations. +func (o *ProgressBarObserver) ProgressBarForSteps(steps int64, description string) *progressbar.ProgressBar { + if o.quiet { + return progressbar.DefaultSilent(steps, description) + } + return progressbar.NewOptions64(steps, + progressbar.OptionSetDescription(description), + progressbar.OptionShowCount(), + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionClearOnFinish(), + ) +} diff --git a/cmd/root.go b/cmd/root.go index 46e72fde..c9bf586a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -219,6 +219,7 @@ func GetAnalyzer(ctx context.Context, command string) (*analyze.Analyzer, error) gitClient := gitops.NewGitClient(nil) analyzer := analyze.NewAnalyzer(scmClient, gitClient, formatter, config, opaClient) + analyzer.Observer = analyze.NewProgressBarObserver(config.Quiet) return analyzer, nil } @@ -239,6 +240,7 @@ func GetAnalyzerWithConfig(ctx context.Context, command, scmProvider, scmBaseURL gitClient := gitops.NewGitClient(nil) analyzer := analyze.NewAnalyzer(scmClient, gitClient, formatter, cfg, opaClient) + analyzer.Observer = analyze.NewProgressBarObserver(cfg.Quiet) return analyzer, nil } From e78a61f6e19abf5a26982375e44790679227d4bf Mon Sep 17 00:00:00 2001 From: Alexis-Maurer Fortin Date: Wed, 8 Apr 2026 09:20:51 -0400 Subject: [PATCH 2/5] addressed comments --- analyze/analyze.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/analyze/analyze.go b/analyze/analyze.go index 46e50830..c4689b59 100644 --- a/analyze/analyze.go +++ b/analyze/analyze.go @@ -145,12 +145,14 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine } }() + discoveryCompleted := false for repoBatch := range orgReposBatches { if repoBatch.Err != nil { log.Error().Err(repoBatch.Err).Msg("failed to fetch batch of repos, skipping batch") continue } - if repoBatch.TotalCount != 0 { + if !discoveryCompleted && repoBatch.TotalCount != 0 { + discoveryCompleted = true obs.OnDiscoveryCompleted(org, repoBatch.TotalCount) } @@ -242,7 +244,6 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine obs.OnFinalizeStarted(len(scannedPackages)) err = a.finalizeAnalysis(ctx, scannedPackages) if err != nil { - obs.OnFinalizeCompleted() return scannedPackages, err } obs.OnFinalizeCompleted() @@ -281,6 +282,7 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, repoUrl := repo.BuildGitURL(a.ScmClient.GetProviderBaseURL()) repoKey, err := a.fetchCone(ctx, repoUrl, a.ScmClient.GetToken(), "refs/heads/*:refs/remotes/origin/*", ".github/workflows") if err != nil { + obs.OnRepoError(repoString, err) return nil, fmt.Errorf("failed to fetch cone: %w", err) } defer a.GitClient.Cleanup(repoKey) @@ -290,6 +292,7 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, workflows, err := a.GitClient.GetUniqWorkflowsBranches(ctx, repoKey) if err != nil { + obs.OnRepoError(repoString, err) return nil, fmt.Errorf("failed to get unique workflow: %w", err) } @@ -355,6 +358,7 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, close(filesChan) wgConsumer.Wait() for err := range errChan { + obs.OnRepoError(repoString, err) return nil, err } @@ -362,6 +366,7 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, _ = bar.Add(1) pkg, err := a.GeneratePackageInsights(ctx, repoKey, repo, "HEAD") if err != nil { + obs.OnRepoError(repoString, err) return nil, fmt.Errorf("failed to generate package insight: %w", err) } @@ -374,6 +379,7 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, scannedPackage, err := inventory.ScanPackageScanner(ctx, *pkg, &inventoryScanner) if err != nil { + obs.OnRepoError(repoString, err) return nil, fmt.Errorf("failed to scan package: %w", err) } @@ -403,7 +409,6 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, scannedPackage.FindingsResults.Findings = expanded if err := a.Formatter.Format(ctx, []*models.PackageInsights{scannedPackage}); err != nil { - obs.OnFinalizeCompleted() return nil, fmt.Errorf("failed to finalize analysis of package: %w", err) } } else { @@ -417,7 +422,6 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, } if err := a.Formatter.FormatWithPath(ctx, []*models.PackageInsights{scannedPackage}, results); err != nil { - obs.OnFinalizeCompleted() return nil, fmt.Errorf("failed to finalize analysis of package: %w", err) } } @@ -455,6 +459,7 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin obs.OnRepoStarted(repoString) repoKey, err := a.cloneRepo(ctx, repo.BuildGitURL(a.ScmClient.GetProviderBaseURL()), a.ScmClient.GetToken(), ref) if err != nil { + obs.OnRepoError(repoString, err) return nil, err } defer a.GitClient.Cleanup(repoKey) @@ -464,11 +469,13 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin pkg, err := a.GeneratePackageInsights(ctx, repoKey, repo, ref) if err != nil { + obs.OnRepoError(repoString, err) return nil, err } files, err := a.GitClient.ListFiles(repoKey, []string{".yml", ".yaml"}) if err != nil { + obs.OnRepoError(repoString, err) return nil, fmt.Errorf("failed to list files: %w", err) } @@ -485,6 +492,7 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin scannedPackage, err := inventory.ScanPackageScanner(ctx, *pkg, memScanner) if err != nil { + obs.OnRepoError(repoString, err) return nil, err } _ = bar.Finish() @@ -493,7 +501,6 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin obs.OnFinalizeStarted(1) err = a.finalizeAnalysis(ctx, []*models.PackageInsights{scannedPackage}) if err != nil { - obs.OnFinalizeCompleted() return nil, err } obs.OnFinalizeCompleted() From 2c12991df2a49ff164965aaef06a8bd4740760a2 Mon Sep 17 00:00:00 2001 From: Alexis-Maurer Fortin Date: Wed, 8 Apr 2026 10:50:19 -0400 Subject: [PATCH 3/5] as it should have been removed progress bar entirely from analysis. Now everything goes through the observer interface --- analyze/analyze.go | 40 +++------------ analyze/analyze_test.go | 12 +++++ analyze/observer.go | 14 ++++++ analyze/progress_bar_observer.go | 86 +++++++++++++++++++++----------- 4 files changed, 91 insertions(+), 61 deletions(-) diff --git a/analyze/analyze.go b/analyze/analyze.go index c4689b59..eb8944ef 100644 --- a/analyze/analyze.go +++ b/analyze/analyze.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "os" "path/filepath" "regexp" "strings" @@ -22,7 +21,6 @@ import ( scm_domain "github.com/boostsecurityio/poutine/providers/scm/domain" "github.com/boostsecurityio/poutine/scanner" "github.com/rs/zerolog/log" - "github.com/schollz/progressbar/v3" ) const TEMP_DIR_PREFIX = "poutine-*" @@ -122,8 +120,9 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine pkgsupplyClient := pkgsupply.NewStaticClient() inventory := scanner.NewInventory(a.Opa, pkgsupplyClient, provider, providerVersion) - log.Debug().Msgf("Starting repository analysis for organization: %s on %s", org, provider) obs := a.observer() + obs.OnAnalysisStarted("Discovering repositories") + log.Debug().Msgf("Starting repository analysis for organization: %s on %s", org, provider) var reposWg sync.WaitGroup errChan := make(chan error, 1) @@ -274,9 +273,8 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, inventory := scanner.NewInventory(a.Opa, pkgsupplyClient, provider, providerVersion) obs := a.observer() + obs.OnAnalysisStarted("Cloning repository") log.Debug().Msgf("Starting repository analysis for: %s/%s on %s", org, repoName, provider) - bar := a.ProgressBar(3, "Cloning repository") - _ = bar.RenderBlank() obs.OnRepoStarted(repoString) repoUrl := repo.BuildGitURL(a.ScmClient.GetProviderBaseURL()) @@ -287,8 +285,7 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, } defer a.GitClient.Cleanup(repoKey) - bar.Describe("Listing unique workflows") - _ = bar.Add(1) + obs.OnStepCompleted("Listing unique workflows") workflows, err := a.GitClient.GetUniqWorkflowsBranches(ctx, repoKey) if err != nil { @@ -296,8 +293,7 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, return nil, fmt.Errorf("failed to get unique workflow: %w", err) } - bar.Describe("Check which workflows match regex: " + regex.String()) - _ = bar.Add(1) + obs.OnStepCompleted("Checking workflows match regex: " + regex.String()) errChan := make(chan error, 1) maxGoroutines := 5 @@ -362,8 +358,7 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, return nil, err } - bar.Describe("Scanning package") - _ = bar.Add(1) + obs.OnStepCompleted("Scanning package") pkg, err := a.GeneratePackageInsights(ctx, repoKey, repo, "HEAD") if err != nil { obs.OnRepoError(repoString, err) @@ -383,7 +378,6 @@ func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, return nil, fmt.Errorf("failed to scan package: %w", err) } - _ = bar.Finish() obs.OnRepoCompleted(repoString, scannedPackage) obs.OnFinalizeStarted(1) @@ -452,9 +446,8 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin inventory := scanner.NewInventory(a.Opa, pkgsupplyClient, provider, providerVersion) obs := a.observer() + obs.OnAnalysisStarted("Cloning repository") log.Debug().Msgf("Starting repository analysis for: %s/%s on %s", org, repoName, provider) - bar := a.ProgressBar(2, "Cloning repository") - _ = bar.RenderBlank() obs.OnRepoStarted(repoString) repoKey, err := a.cloneRepo(ctx, repo.BuildGitURL(a.ScmClient.GetProviderBaseURL()), a.ScmClient.GetToken(), ref) @@ -464,8 +457,7 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin } defer a.GitClient.Cleanup(repoKey) - bar.Describe("Analyzing repository") - _ = bar.Add(1) + obs.OnStepCompleted("Cloned repository") pkg, err := a.GeneratePackageInsights(ctx, repoKey, repo, ref) if err != nil { @@ -495,7 +487,6 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin obs.OnRepoError(repoString, err) return nil, err } - _ = bar.Finish() obs.OnRepoCompleted(repoString, scannedPackage) obs.OnFinalizeStarted(1) @@ -754,18 +745,3 @@ func (a *Analyzer) cloneRepo(ctx context.Context, gitURL string, token string, r } return key, nil } - -func (a *Analyzer) ProgressBar(maxValue int64, description string) *progressbar.ProgressBar { - if a.Config.Quiet { - return progressbar.DefaultSilent(maxValue, description) - } else { - return progressbar.NewOptions64( - maxValue, - progressbar.OptionSetDescription(description), - progressbar.OptionShowCount(), - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionClearOnFinish(), - ) - - } -} diff --git a/analyze/analyze_test.go b/analyze/analyze_test.go index d35f84cf..dc95a3a9 100644 --- a/analyze/analyze_test.go +++ b/analyze/analyze_test.go @@ -197,6 +197,11 @@ type mockObserver struct { events []string } +func (m *mockObserver) OnAnalysisStarted(description string) { + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, "analysis_started:"+description) +} func (m *mockObserver) OnDiscoveryCompleted(org string, totalCount int) { m.mu.Lock() defer m.mu.Unlock() @@ -222,6 +227,11 @@ func (m *mockObserver) OnRepoSkipped(repo string, reason string) { defer m.mu.Unlock() m.events = append(m.events, fmt.Sprintf("repo_skipped:%s:%s", repo, reason)) } +func (m *mockObserver) OnStepCompleted(description string) { + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, "step_completed:"+description) +} func (m *mockObserver) OnFinalizeStarted(total int) { m.mu.Lock() defer m.mu.Unlock() @@ -261,6 +271,7 @@ func TestProgressObserverInterface(t *testing.T) { obs.OnRepoCompleted("test-org/repo1", nil) obs.OnRepoSkipped("test-org/repo2", "fork") obs.OnRepoError("test-org/repo3", errors.New("clone failed")) + obs.OnStepCompleted("Analyzing repository") obs.OnFinalizeStarted(1) obs.OnFinalizeCompleted() @@ -270,6 +281,7 @@ func TestProgressObserverInterface(t *testing.T) { "repo_completed:test-org/repo1", "repo_skipped:test-org/repo2:fork", "repo_error:test-org/repo3", + "step_completed:Analyzing repository", "finalize_started:1", "finalize_completed", }, obs.events) diff --git a/analyze/observer.go b/analyze/observer.go index ed3d4df0..199eb917 100644 --- a/analyze/observer.go +++ b/analyze/observer.go @@ -11,6 +11,7 @@ import "github.com/boostsecurityio/poutine/models" // During AnalyzeOrg, methods are called from two contexts: // // Main goroutine (sequential, never concurrent with each other): +// - OnAnalysisStarted // - OnDiscoveryCompleted // - OnRepoSkipped // - OnFinalizeStarted @@ -28,6 +29,11 @@ import "github.com/boostsecurityio/poutine/models" // During AnalyzeRepo and AnalyzeStaleBranches all methods are called // from a single goroutine. type ProgressObserver interface { + // OnAnalysisStarted is called from the main goroutine at the very + // beginning of any analysis, before any work begins. Useful for + // rendering an initial status indicator while discovery or cloning runs. + OnAnalysisStarted(description string) + // OnDiscoveryCompleted is called from the main goroutine when the // total repo count is known (first batch with TotalCount > 0). OnDiscoveryCompleted(org string, totalCount int) @@ -48,6 +54,12 @@ type ProgressObserver interface { // skipped (fork, empty, etc.). OnRepoSkipped(repo string, reason string) + // OnStepCompleted is called when a sub-step within a single-repo + // analysis completes (e.g. "Cloning repository", "Analyzing repository"). + // Used by AnalyzeRepo and AnalyzeStaleBranches for step-level progress. + // Called from the main goroutine. + OnStepCompleted(description string) + // OnFinalizeStarted is called from the main goroutine when the // formatting/output phase begins, after all repos are processed. OnFinalizeStarted(totalPackages int) @@ -59,10 +71,12 @@ type ProgressObserver interface { type noopObserver struct{} +func (noopObserver) OnAnalysisStarted(string) {} func (noopObserver) OnDiscoveryCompleted(string, int) {} func (noopObserver) OnRepoStarted(string) {} func (noopObserver) OnRepoCompleted(string, *models.PackageInsights) {} func (noopObserver) OnRepoError(string, error) {} func (noopObserver) OnRepoSkipped(string, string) {} +func (noopObserver) OnStepCompleted(string) {} func (noopObserver) OnFinalizeStarted(int) {} func (noopObserver) OnFinalizeCompleted() {} diff --git a/analyze/progress_bar_observer.go b/analyze/progress_bar_observer.go index b46e0c2d..e89ade11 100644 --- a/analyze/progress_bar_observer.go +++ b/analyze/progress_bar_observer.go @@ -8,59 +8,87 @@ import ( ) // ProgressBarObserver implements ProgressObserver by rendering a CLI progress bar. +// For org analysis it shows a repo-count bar (created on OnDiscoveryCompleted). +// For single-repo analysis it shows a step-level bar (created on first OnStepCompleted). type ProgressBarObserver struct { bar *progressbar.ProgressBar quiet bool } func NewProgressBarObserver(quiet bool) *ProgressBarObserver { - var bar *progressbar.ProgressBar - if quiet { - bar = progressbar.DefaultSilent(0, "Analyzing repositories") - } else { - bar = progressbar.NewOptions64(0, - progressbar.OptionSetDescription("Analyzing repositories"), - progressbar.OptionShowCount(), - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionClearOnFinish(), - ) + return &ProgressBarObserver{quiet: quiet} +} + +func (o *ProgressBarObserver) newBar(max int64, description string) *progressbar.ProgressBar { + if o.quiet { + return progressbar.DefaultSilent(max, description) } - return &ProgressBarObserver{bar: bar, quiet: quiet} + return progressbar.NewOptions64(max, + progressbar.OptionSetDescription(description), + progressbar.OptionShowCount(), + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionClearOnFinish(), + ) +} + +func (o *ProgressBarObserver) OnAnalysisStarted(description string) { + // Create an indeterminate spinner bar (max=-1) to show activity + // before the total count is known or the first step completes. + o.bar = o.newBar(-1, description) + _ = o.bar.RenderBlank() } func (o *ProgressBarObserver) OnDiscoveryCompleted(_ string, totalCount int) { - o.bar.ChangeMax(totalCount) + // Finish the spinner before replacing with a counting bar. + if o.bar != nil { + _ = o.bar.Finish() + } + o.bar = o.newBar(int64(totalCount), "Analyzing repositories") } func (o *ProgressBarObserver) OnRepoStarted(_ string) {} func (o *ProgressBarObserver) OnRepoCompleted(_ string, _ *models.PackageInsights) { - _ = o.bar.Add(1) + if o.bar != nil { + _ = o.bar.Add(1) + } } func (o *ProgressBarObserver) OnRepoError(_ string, _ error) { - _ = o.bar.Add(1) + if o.bar != nil { + _ = o.bar.Add(1) + } } func (o *ProgressBarObserver) OnRepoSkipped(_ string, _ string) { - o.bar.ChangeMax(o.bar.GetMax() - 1) + if o.bar != nil { + if newMax := o.bar.GetMax() - 1; newMax >= 0 { + o.bar.ChangeMax(newMax) + } + } } -func (o *ProgressBarObserver) OnFinalizeStarted(_ int) { - _ = o.bar.Finish() +func (o *ProgressBarObserver) OnStepCompleted(description string) { + if o.bar != nil && o.bar.GetMax64() == -1 { + // Finish the spinner, then create a step bar. + _ = o.bar.Finish() + o.bar = nil + } + if o.bar == nil { + // First step — create the bar. + o.bar = o.newBar(1, description) + } else { + // Grow the bar to accommodate each new step, then advance. + o.bar.ChangeMax(o.bar.GetMax() + 1) + o.bar.Describe(description) + } + _ = o.bar.Add(1) } -func (o *ProgressBarObserver) OnFinalizeCompleted() {} - -// ProgressBarForSteps creates a step-counting progress bar for single-repo operations. -func (o *ProgressBarObserver) ProgressBarForSteps(steps int64, description string) *progressbar.ProgressBar { - if o.quiet { - return progressbar.DefaultSilent(steps, description) +func (o *ProgressBarObserver) OnFinalizeStarted(_ int) { + if o.bar != nil { + _ = o.bar.Finish() } - return progressbar.NewOptions64(steps, - progressbar.OptionSetDescription(description), - progressbar.OptionShowCount(), - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionClearOnFinish(), - ) } + +func (o *ProgressBarObserver) OnFinalizeCompleted() {} From 1dc9b79ef305f760218672cd670d3510a1d2bac8 Mon Sep 17 00:00:00 2001 From: Alexis-Maurer Fortin Date: Wed, 8 Apr 2026 10:57:45 -0400 Subject: [PATCH 4/5] lint --- analyze/progress_bar_observer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/analyze/progress_bar_observer.go b/analyze/progress_bar_observer.go index e89ade11..1c59a8c8 100644 --- a/analyze/progress_bar_observer.go +++ b/analyze/progress_bar_observer.go @@ -19,11 +19,11 @@ func NewProgressBarObserver(quiet bool) *ProgressBarObserver { return &ProgressBarObserver{quiet: quiet} } -func (o *ProgressBarObserver) newBar(max int64, description string) *progressbar.ProgressBar { +func (o *ProgressBarObserver) newBar(barMax int64, description string) *progressbar.ProgressBar { if o.quiet { - return progressbar.DefaultSilent(max, description) + return progressbar.DefaultSilent(barMax, description) } - return progressbar.NewOptions64(max, + return progressbar.NewOptions64(barMax, progressbar.OptionSetDescription(description), progressbar.OptionShowCount(), progressbar.OptionSetWriter(os.Stderr), From fc05c01351db427f42d6b606485f9234fe71d142 Mon Sep 17 00:00:00 2001 From: Alexis-Maurer Fortin Date: Wed, 8 Apr 2026 11:49:18 -0400 Subject: [PATCH 5/5] better step vs repos for bar pogression management --- analyze/analyze.go | 4 ++++ analyze/progress_bar_observer.go | 10 ++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/analyze/analyze.go b/analyze/analyze.go index eb8944ef..7a91c067 100644 --- a/analyze/analyze.go +++ b/analyze/analyze.go @@ -465,6 +465,8 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin return nil, err } + obs.OnStepCompleted("Generated package insights") + files, err := a.GitClient.ListFiles(repoKey, []string{".yml", ".yaml"}) if err != nil { obs.OnRepoError(repoString, err) @@ -487,6 +489,8 @@ func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref strin obs.OnRepoError(repoString, err) return nil, err } + + obs.OnStepCompleted("Scanned repository") obs.OnRepoCompleted(repoString, scannedPackage) obs.OnFinalizeStarted(1) diff --git a/analyze/progress_bar_observer.go b/analyze/progress_bar_observer.go index 1c59a8c8..1d149ddb 100644 --- a/analyze/progress_bar_observer.go +++ b/analyze/progress_bar_observer.go @@ -11,8 +11,9 @@ import ( // For org analysis it shows a repo-count bar (created on OnDiscoveryCompleted). // For single-repo analysis it shows a step-level bar (created on first OnStepCompleted). type ProgressBarObserver struct { - bar *progressbar.ProgressBar - quiet bool + bar *progressbar.ProgressBar + quiet bool + stepMode bool // true when bar is step-driven (single-repo analysis) } func NewProgressBarObserver(quiet bool) *ProgressBarObserver { @@ -49,13 +50,13 @@ func (o *ProgressBarObserver) OnDiscoveryCompleted(_ string, totalCount int) { func (o *ProgressBarObserver) OnRepoStarted(_ string) {} func (o *ProgressBarObserver) OnRepoCompleted(_ string, _ *models.PackageInsights) { - if o.bar != nil { + if o.bar != nil && !o.stepMode { _ = o.bar.Add(1) } } func (o *ProgressBarObserver) OnRepoError(_ string, _ error) { - if o.bar != nil { + if o.bar != nil && !o.stepMode { _ = o.bar.Add(1) } } @@ -69,6 +70,7 @@ func (o *ProgressBarObserver) OnRepoSkipped(_ string, _ string) { } func (o *ProgressBarObserver) OnStepCompleted(description string) { + o.stepMode = true if o.bar != nil && o.bar.GetMax64() == -1 { // Finish the spinner, then create a step bar. _ = o.bar.Finish()