From 022d56dc8e388dc1f855baaaafbd0326101656f8 Mon Sep 17 00:00:00 2001 From: Urko Date: Tue, 18 Jul 2023 12:05:22 +0200 Subject: [PATCH] feat: handle multiple commands sync and list duplicates --- cmd/duplicate_versions.go | 52 ++++++++++++++++++++++++++++ cmd/{main.go => sync.go} | 28 +++++---------- internal/services/backblaze.go | 63 ++++++++++++++++++++++++++++++++-- main.go | 29 ++++++++++++++++ 4 files changed, 151 insertions(+), 21 deletions(-) create mode 100644 cmd/duplicate_versions.go rename cmd/{main.go => sync.go} (61%) create mode 100644 main.go diff --git a/cmd/duplicate_versions.go b/cmd/duplicate_versions.go new file mode 100644 index 0000000..41cfa23 --- /dev/null +++ b/cmd/duplicate_versions.go @@ -0,0 +1,52 @@ +package cmd + +import ( + "context" + "log" + "os" + "os/signal" + "syscall" + + "gitea.urkob.com/urko/backblaze-backup/internal/services" + "gitea.urkob.com/urko/backblaze-backup/kit/config" + "github.com/spf13/cobra" +) + +var Versions = &cobra.Command{ + Use: "versions", + Short: "Handle versions of files in Backblaze", + Run: func(cmd *cobra.Command, args []string) { + log.SetFlags(log.Lmicroseconds) + ctx, cancel := context.WithCancel(signalContext(cmd.Context())) + defer cancel() + + envFile := "" + if os.Getenv("BACKBLAZE_ENV") == "dev" { + envFile = ".env" + } + cfg := config.NewConfig(envFile) + + bbService := services.NewBackBlaze(cfg.BbId, cfg.BbKey) + if err := bbService.ListDuplicateVersions(ctx); err != nil { + log.Fatalln("bbService.ListDuplicateVersions()", err) + } + }, +} + +func signalContext(ctx context.Context) context.Context { + ctx, cancel := context.WithCancel(ctx) + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + + go func() { + log.Println("listening for shutdown signal") + <-sigs + log.Println("shutdown signal received") + signal.Stop(sigs) + close(sigs) + cancel() + }() + + return ctx +} diff --git a/cmd/main.go b/cmd/sync.go similarity index 61% rename from cmd/main.go rename to cmd/sync.go index e5bb91e..ca8e9bf 100644 --- a/cmd/main.go +++ b/cmd/sync.go @@ -1,7 +1,7 @@ -package main +package cmd import ( - "fmt" + "context" "log" "os" @@ -10,11 +10,14 @@ import ( "github.com/spf13/cobra" ) -var rootCmd = &cobra.Command{ - Use: "backblaze-backup", - Short: "Backblaze backup tool", +var Sync = &cobra.Command{ + Use: "sync", + Short: "Sync files or directories to Backblaze", Long: `A tool to backup files and directories to Backblaze.`, Run: func(cmd *cobra.Command, args []string) { + ctx, cancel := context.WithCancel(signalContext(cmd.Context())) + defer cancel() + log.SetFlags(log.Lmicroseconds) filePath, err := cmd.Flags().GetString("file") if err != nil { @@ -36,21 +39,8 @@ var rootCmd = &cobra.Command{ cfg := config.NewConfig(envFile) bbService := services.NewBackBlaze(cfg.BbId, cfg.BbKey).WithBucket(bucketName).WithDir(dir).WithFile(filePath) - if err := bbService.Sync(); err != nil { + if err := bbService.Sync(ctx); err != nil { log.Fatalln("bbService.Sync()", err) } }, } - -func init() { - rootCmd.PersistentFlags().String("file", "", "absolute path of the file you want to upload to backblaze") - rootCmd.PersistentFlags().String("dir", "", "absolute path of the directory you want to upload to backblaze") - rootCmd.PersistentFlags().String("bucket", "", "backblaze bucket name") -} - -func main() { - if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } -} diff --git a/internal/services/backblaze.go b/internal/services/backblaze.go index ba47e04..c944641 100644 --- a/internal/services/backblaze.go +++ b/internal/services/backblaze.go @@ -50,7 +50,7 @@ func (b *BackBalze) WithFile(filePath string) *BackBalze { b.filePath = filePath return b } -func (b *BackBalze) Sync() error { +func (b *BackBalze) Sync(ctx context.Context) error { if b.bucketName == "" && (b.filePath == "" || b.dir == "") { return fmt.Errorf("bucket name is %v | filePath is %v | dir is %v", b.bucketName, b.filePath, b.dir) } @@ -59,7 +59,6 @@ func (b *BackBalze) Sync() error { return errors.New("you must select just 1 option, dir or file") } - ctx := context.Background() b2Client, err := b2.NewClient(ctx, b.bbID, b.bbKey) if err != nil { return fmt.Errorf("b2.NewClient %w", err) @@ -302,3 +301,63 @@ func bucketFiles(ctx context.Context, bucket *b2.Bucket) ([]string, error) { } return files, nil } + +type duplicate struct { + bucket string + file string + count int +} + +func (b *BackBalze) ListDuplicateVersions(ctx context.Context) error { + b2Client, err := b2.NewClient(ctx, b.bbID, b.bbKey) + if err != nil { + return fmt.Errorf("b2.NewClient %w", err) + } + log.Println("b2Client ok") + + buckets, err := b2Client.ListBuckets(ctx) + if err != nil { + return fmt.Errorf("b2Client.Bucket %w", err) + } + + dups := make([]duplicate, 0) + + log.Println("len(buckets)", len(buckets)) + for _, bc := range buckets { + files := make(map[string]int, 0) + + bucketIter := bc.List(ctx, b2.ListHidden()) + if bucketIter == nil { + return fmt.Errorf("bucket list cannot be nil") + } + + for { + if !bucketIter.Next() { + if bucketIter.Err() != nil { + return fmt.Errorf("bucketIter err %w", bucketIter.Err()) + } + break + } + if bucketIter.Object() == nil { + log.Println("bucketIter Object is nil") + continue + } + files[bucketIter.Object().Name()]++ + } + + // Search duplicates + for file, count := range files { + if count > 1 { + dups = append(dups, duplicate{ + bucket: bc.Name(), + file: file, + count: count, + }) + } + } + } + if len(dups) > 0 { + return fmt.Errorf("found duplicates: %+v", dups) + } + return nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..78d19b5 --- /dev/null +++ b/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + "os" + + "gitea.urkob.com/urko/backblaze-backup/cmd" + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "backblaze-backup", + Short: "Backblaze backup tool", + Long: `A tool to backup files and directories to Backblaze.`, +} + +func init() { + rootCmd.AddCommand(cmd.Sync, cmd.Versions) + cmd.Sync.PersistentFlags().String("file", "", "absolute path of the file you want to upload to backblaze") + cmd.Sync.PersistentFlags().String("dir", "", "absolute path of the directory you want to upload to backblaze") + cmd.Sync.PersistentFlags().String("bucket", "", "backblaze bucket name") +} + +func main() { + if err := cmd.Sync.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +}