diff --git a/Makefile b/Makefile index b0ce12a..11ad66e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ BIN_DIR := ./bin APP_NAME := backblaze-backup -MAIN := ./cmd/main.go +MAIN := ./main.go .PHONY: local_run local_run: diff --git a/cmd/duplicate_versions.go b/cmd/duplicate_versions.go index 41cfa23..25ee783 100644 --- a/cmd/duplicate_versions.go +++ b/cmd/duplicate_versions.go @@ -27,7 +27,7 @@ var Versions = &cobra.Command{ cfg := config.NewConfig(envFile) bbService := services.NewBackBlaze(cfg.BbId, cfg.BbKey) - if err := bbService.ListDuplicateVersions(ctx); err != nil { + if err := bbService.ListDuplicateVersions(ctx, cancel); err != nil { log.Fatalln("bbService.ListDuplicateVersions()", err) } }, diff --git a/internal/services/backblaze.go b/internal/services/backblaze.go index c944641..bc61cd1 100644 --- a/internal/services/backblaze.go +++ b/internal/services/backblaze.go @@ -308,7 +308,7 @@ type duplicate struct { count int } -func (b *BackBalze) ListDuplicateVersions(ctx context.Context) error { +func (b *BackBalze) ListDuplicateVersions(ctx context.Context, cancel context.CancelFunc) error { b2Client, err := b2.NewClient(ctx, b.bbID, b.bbKey) if err != nil { return fmt.Errorf("b2.NewClient %w", err) @@ -320,44 +320,63 @@ func (b *BackBalze) ListDuplicateVersions(ctx context.Context) error { return fmt.Errorf("b2Client.Bucket %w", err) } + wg := sync.WaitGroup{} dups := make([]duplicate, 0) log.Println("len(buckets)", len(buckets)) + sm := semaphore.NewWeighted(int64(b.maxWorkers)) + wg.Add(len(buckets)) for _, bc := range buckets { - files := make(map[string]int, 0) - - bucketIter := bc.List(ctx, b2.ListHidden()) - if bucketIter == nil { - return fmt.Errorf("bucket list cannot be nil") + if err := sm.Acquire(ctx, 1); err != nil { + return fmt.Errorf("sm.Acquire %w", err) } - for { - if !bucketIter.Next() { - if bucketIter.Err() != nil { - return fmt.Errorf("bucketIter err %w", bucketIter.Err()) + go func(bc *b2.Bucket) { + defer sm.Release(1) + defer wg.Done() + files := make(map[string]int, 0) + + bucketIter := bc.List(ctx, b2.ListHidden()) + if bucketIter == nil { + log.Println("bucket list cannot be nil") + return + } + + for { + if !bucketIter.Next() { + if bucketIter.Err() != nil { + log.Println("bucketIter err %w", bucketIter.Err()) + return + } + break } - break + if bucketIter.Object() == nil { + log.Println("bucketIter Object is nil") + continue + } + files[bucketIter.Object().Name()]++ } - if bucketIter.Object() == nil { - log.Println("bucketIter Object is nil") - continue - } - files[bucketIter.Object().Name()]++ - } - // Search duplicates - for file, count := range files { - if count > 1 { - dups = append(dups, duplicate{ - bucket: bc.Name(), - file: file, - count: count, - }) + // Search duplicates + for file, count := range files { + if count > 1 { + dups = append(dups, duplicate{ + bucket: bc.Name(), + file: file, + count: count, + }) + } } - } + }(bc) } + wg.Wait() + if len(dups) > 0 { - return fmt.Errorf("found duplicates: %+v", dups) + var builder strings.Builder + for _, dup := range dups { + builder.WriteString(fmt.Sprintf("%+v\n", dup)) + } + return fmt.Errorf("found duplicates: %s", builder.String()) } return nil }