Skip to content

Commit 510964b

Browse files
Amazing-StardomAmazing-Stardom
authored
Merge pull request #121 from HexmosTech/b2m-v3
Implement BLAKE3 Caching and Refactor Core Rclone Utilities
2 parents bf8e79f + f4bd2cb commit 510964b

30 files changed

Lines changed: 1738 additions & 996 deletions

‎.gitignore‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -162,9 +162,9 @@ frontend/assets/js/*
162162

163163
# sitemap-checker
164164
frontend/scripts/sitemap-checker/sitemap-checker
165-
frontend/b2m.toml
166165
frontend/bun.lock
167166
frontend/b2m
168167
b2m
169-
b2-manager/b2m.toml
170-
b2-manager/db/all_dbs/version/
168+
b2-manager/.b2m/
169+
b2-manager/fdt-dev.toml
170+
frontend/.b2m/

‎b2-manager/config/config.go‎

Lines changed: 80 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -8,105 +8,40 @@ import (
88
"strings"
99

1010
"github.com/BurntSushi/toml"
11-
)
12-
13-
// Config holds all application configuration
14-
type Config struct {
15-
// Paths
16-
RootBucket string
17-
LockDir string
18-
VersionDir string
19-
LocalVersionDir string
20-
LocalAnchorDir string
21-
LocalDBDir string
22-
23-
// Environment
24-
DiscordWebhookURL string
25-
26-
// User Info
27-
CurrentUser string
28-
Hostname string
29-
ProjectRoot string
30-
31-
// Tool Info
32-
ToolVersion string
33-
}
3411

35-
var AppConfig = Config{
36-
ToolVersion: "v1.0",
37-
}
38-
39-
// Sync Status Constants
40-
const (
41-
SyncStatusLocalOnly = "+"
42-
SyncStatusRemoteOnly = "-"
43-
SyncStatusDifferent = "*"
12+
"b2m/model"
4413
)
4514

4615
// InitializeConfig sets up global configuration variables
4716
func InitializeConfig() error {
4817
var err error
4918

50-
AppConfig.ProjectRoot, err = findProjectRoot()
19+
model.AppConfig.ProjectRoot, err = findProjectRoot()
5120
if err != nil {
5221
fmt.Fprintf(os.Stderr, "⚠️ Could not determine project root: %v. Using CWD.\n", err)
53-
AppConfig.ProjectRoot, _ = os.Getwd()
54-
}
55-
56-
// Load config from b2m.toml
57-
tomlPath := filepath.Join(AppConfig.ProjectRoot, "b2m.toml")
58-
if _, err := os.Stat(tomlPath); os.IsNotExist(err) {
59-
return fmt.Errorf("couldn't find b2m.toml file at %s: %w", tomlPath, err)
60-
}
61-
62-
var tomlConf struct {
63-
Discord string `toml:"discord"`
64-
RootBucket string `toml:"rootbucket"`
22+
model.AppConfig.ProjectRoot, _ = os.Getwd()
6523
}
66-
if _, err := toml.DecodeFile(tomlPath, &tomlConf); err != nil {
67-
return fmt.Errorf("failed to decode b2m.toml: %w", err)
68-
}
69-
70-
AppConfig.RootBucket = tomlConf.RootBucket
71-
AppConfig.DiscordWebhookURL = tomlConf.Discord
7224

73-
if AppConfig.RootBucket == "" {
74-
return fmt.Errorf("rootbucket not defined in b2m.toml file")
75-
}
76-
if AppConfig.DiscordWebhookURL == "" {
77-
return fmt.Errorf("discord not defined in b2m.toml file")
25+
// Load config from fdt-dev.toml
26+
if err := loadTOMLConfig(); err != nil {
27+
return err
7828
}
7929

80-
// Derived paths
81-
// Ensure RootBucket ends with /
82-
if !strings.HasSuffix(AppConfig.RootBucket, "/") {
83-
AppConfig.RootBucket += "/"
30+
// Validate and set derived paths
31+
if err := validateAndSetPaths(); err != nil {
32+
return err
8433
}
8534

86-
AppConfig.LockDir = AppConfig.RootBucket + "lock/"
87-
AppConfig.VersionDir = AppConfig.RootBucket + "version/"
35+
// Fetch user details
36+
fetchUserDetails()
8837

89-
var u *user.User
90-
u, err = user.Current()
91-
if err != nil {
92-
AppConfig.CurrentUser = "unknown"
93-
} else {
94-
AppConfig.CurrentUser = u.Username
95-
}
96-
97-
var h string
98-
h, err = os.Hostname()
99-
if err != nil {
100-
AppConfig.Hostname = "unknown"
101-
} else {
102-
AppConfig.Hostname = h
38+
if model.AppConfig.LocalDBDir == "" {
39+
return fmt.Errorf("LocalDBDir not configured. Please set b2m_db_dir in your config file")
10340
}
41+
model.AppConfig.LocalB2MDir = filepath.Join(model.AppConfig.ProjectRoot, ".b2m")
42+
model.AppConfig.LocalVersionDir = filepath.Join(model.AppConfig.LocalB2MDir, "version")
43+
model.AppConfig.LocalAnchorDir = filepath.Join(model.AppConfig.LocalB2MDir, "local-version")
10444

105-
AppConfig.LocalDBDir = filepath.Join(AppConfig.ProjectRoot, "db", "all_dbs")
106-
AppConfig.LocalVersionDir = filepath.Join(AppConfig.ProjectRoot, "db", "all_dbs", "version")
107-
AppConfig.LocalAnchorDir = filepath.Join(AppConfig.ProjectRoot, "db", "all_dbs", "local-version")
108-
109-
// Initialize logging if needed, or other startup tasks
11045
return nil
11146
}
11247

@@ -119,13 +54,73 @@ func findProjectRoot() (string, error) {
11954
if info, err := os.Stat(filepath.Join(dir, "db")); err == nil && info.IsDir() {
12055
return dir, nil
12156
}
122-
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
123-
return dir, nil
124-
}
12557
parent := filepath.Dir(dir)
12658
if parent == dir {
127-
return "", fmt.Errorf("root not found (searched for 'db' dir or 'go.mod')")
59+
return "", fmt.Errorf("root not found 'db' dir")
12860
}
12961
dir = parent
13062
}
13163
}
64+
65+
func loadTOMLConfig() error {
66+
tomlPath := filepath.Join(model.AppConfig.ProjectRoot, "fdt-dev.toml")
67+
if _, err := os.Stat(tomlPath); os.IsNotExist(err) {
68+
return fmt.Errorf("couldn't find fdt-dev.toml file at %s: %w", tomlPath, err)
69+
}
70+
71+
var tomlConf struct {
72+
B2M struct {
73+
Discord string `toml:"b2m_discord_webhook"`
74+
RootBucket string `toml:"b2m_remote_root_bucket"`
75+
LocalDBDir string `toml:"b2m_db_dir"`
76+
} `toml:"b2m"`
77+
}
78+
if _, err := toml.DecodeFile(tomlPath, &tomlConf); err != nil {
79+
return fmt.Errorf("failed to decode fdt-dev.toml: %w", err)
80+
}
81+
82+
model.AppConfig.RootBucket = tomlConf.B2M.RootBucket
83+
model.AppConfig.DiscordWebhookURL = tomlConf.B2M.Discord
84+
if tomlConf.B2M.LocalDBDir != "" {
85+
if filepath.IsAbs(tomlConf.B2M.LocalDBDir) {
86+
model.AppConfig.LocalDBDir = tomlConf.B2M.LocalDBDir
87+
} else {
88+
model.AppConfig.LocalDBDir = filepath.Join(model.AppConfig.ProjectRoot, tomlConf.B2M.LocalDBDir)
89+
}
90+
}
91+
92+
return nil
93+
}
94+
95+
func validateAndSetPaths() error {
96+
if model.AppConfig.RootBucket == "" {
97+
return fmt.Errorf("b2m_remote_root_bucket not defined in fdt-dev.toml file")
98+
}
99+
if model.AppConfig.DiscordWebhookURL == "" {
100+
return fmt.Errorf("b2m_discord_webhook not defined in fdt-dev.toml file")
101+
}
102+
103+
if !strings.HasSuffix(model.AppConfig.RootBucket, "/") {
104+
model.AppConfig.RootBucket += "/"
105+
}
106+
107+
model.AppConfig.LockDir = model.AppConfig.RootBucket + "lock/"
108+
model.AppConfig.VersionDir = model.AppConfig.RootBucket + "version/"
109+
return nil
110+
}
111+
112+
func fetchUserDetails() {
113+
u, err := user.Current()
114+
if err != nil {
115+
model.AppConfig.CurrentUser = "unknown"
116+
} else {
117+
model.AppConfig.CurrentUser = u.Username
118+
}
119+
120+
h, err := os.Hostname()
121+
if err != nil {
122+
model.AppConfig.Hostname = "unknown"
123+
} else {
124+
model.AppConfig.Hostname = h
125+
}
126+
}

‎b2-manager/config/init.go‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
package config
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
8+
"b2m/core"
9+
"b2m/model"
10+
)
11+
12+
// InitSystem initializes the system, handles CLI commands, and returns the signal handler.
13+
// The caller is responsible for calling Cleanup() when done.
14+
func InitSystem() *core.SignalHandler {
15+
// Initialize Logger explicit early to capture startup issues
16+
if err := core.InitLogger(); err != nil {
17+
fmt.Printf("Warning: Failed to initialize logger: %v\n", err)
18+
}
19+
20+
// Initialize Config
21+
if err := InitializeConfig(); err != nil {
22+
core.LogError("Failed to load configuration: %v", err)
23+
fmt.Printf("Error: %v\n", err)
24+
os.Exit(1)
25+
}
26+
27+
// Set version
28+
model.AppConfig.ToolVersion = "2.0"
29+
30+
// Load hash cache
31+
if err := core.LoadHashCache(); err != nil {
32+
core.LogInfo("Warning: Failed to load hash cache: %v", err)
33+
}
34+
35+
core.LogInfo("Configuration loaded successfully")
36+
core.LogInfo("RootBucket: %s", model.AppConfig.RootBucket)
37+
core.LogInfo("DiscordWebhookURL: %s", model.AppConfig.DiscordWebhookURL)
38+
39+
// CLI Command Handling
40+
if len(os.Args) > 1 {
41+
command := os.Args[1]
42+
43+
switch command {
44+
case "--help":
45+
printUsage()
46+
// We exit here because help is a standalone command
47+
os.Exit(0)
48+
49+
case "--generate-hash":
50+
// Common Dependencies check
51+
if err := checkDependencies(); err != nil {
52+
fmt.Printf("Error: %v\n", err)
53+
os.Exit(1)
54+
}
55+
56+
// Warning and Confirmation
57+
fmt.Println("\nWARNING: This operation regenerates all metadata from local files.")
58+
fmt.Println("Ensure your local databases are synced with remote to avoid data loss.")
59+
fmt.Println("This should ONLY be done when changing hashing algorithms or recovering from corruption.")
60+
fmt.Print("\nAre you sure you want to proceed? (y/N): ")
61+
62+
var confirmation string
63+
fmt.Scanln(&confirmation)
64+
if confirmation != "y" && confirmation != "Y" {
65+
fmt.Println("Operation cancelled.")
66+
os.Exit(0)
67+
}
68+
69+
// Clean up .b2m before generating metadata
70+
if err := core.CleanupLocalMetadata(); err != nil {
71+
fmt.Printf("Error: failed to cleanup metadata: %v\n", err)
72+
core.LogError("Generate-Hash: Failed to cleanup metadata: %v", err)
73+
os.Exit(1)
74+
}
75+
76+
// Explicitly clear hash cache
77+
core.ClearHashCache()
78+
79+
// Bootstrap system minimal
80+
// Use background context for CLI tool mode
81+
cliCtx := context.Background()
82+
if err := core.BootstrapSystem(cliCtx); err != nil {
83+
core.LogError("Startup Warning: %v", err)
84+
}
85+
core.HandleBatchMetadataGeneration()
86+
Cleanup()
87+
os.Exit(0)
88+
89+
case "--reset":
90+
fmt.Println("Resetting system state...")
91+
// Clean up .b2m before starting normal execution
92+
if err := core.CleanupLocalMetadata(); err != nil {
93+
fmt.Printf("Error: failed to cleanup metadata: %v\n", err)
94+
core.LogError("Reset: Failed to cleanup metadata: %v", err)
95+
os.Exit(1)
96+
}
97+
98+
// Explicitly clear hash cache
99+
core.ClearHashCache()
100+
101+
Cleanup()
102+
fmt.Println("Reset complete. Please restart the application.")
103+
os.Exit(0)
104+
105+
default:
106+
fmt.Printf("Unknown command: %s\n", command)
107+
printUsage()
108+
os.Exit(1)
109+
}
110+
}
111+
112+
// Setup cancellation handling
113+
sigHandler := core.NewSignalHandler()
114+
115+
// Startup checks for TUI
116+
if err := checkDependencies(); err != nil {
117+
core.LogError("Startup Error: %v", err)
118+
fmt.Printf("Startup Error: %v\n", err)
119+
os.Exit(1)
120+
}
121+
122+
if err := core.BootstrapSystem(sigHandler.Context()); err != nil {
123+
core.LogError("Startup Warning: %v", err)
124+
}
125+
126+
return sigHandler
127+
}
128+
129+
// Cleanup saves the hash cache and closes the logger.
130+
// This should be called (usually deferred) by the main function.
131+
func Cleanup() {
132+
if err := core.SaveHashCache(); err != nil {
133+
core.LogError("Failed to save hash cache: %v", err)
134+
}
135+
core.CloseLogger()
136+
}
137+
138+
func checkDependencies() error {
139+
if err := core.CheckB3SumAvailability(); err != nil {
140+
return err
141+
}
142+
if err := core.CheckRclone(); err != nil {
143+
return fmt.Errorf("rclone not found or error: %w", err)
144+
}
145+
if !core.CheckRcloneConfig() {
146+
return fmt.Errorf("rclone config not found. Run 'init' or check setup")
147+
}
148+
return nil
149+
}
150+
151+
func printUsage() {
152+
fmt.Println("b2-manager - Backblaze B2 Database Manager")
153+
fmt.Println("\nUsage:")
154+
fmt.Println(" b2-manager [command]")
155+
fmt.Println("\nCommands:")
156+
fmt.Println(" --help Show this help message")
157+
fmt.Println(" --generate-hash Generate new hash and create metadata in remote")
158+
fmt.Println(" --reset Remove local metadata caches and start fresh UI session")
159+
fmt.Println("\nIf no command is provided, the TUI application starts normally.")
160+
}

0 commit comments

Comments
 (0)