feat: handle list for repeated files/versions

Makefile

@@ -1,6 +1,6 @@
 BIN_DIR := ./bin
 APP_NAME := backblaze-backup
-MAIN := ./cmd/main.go
+MAIN := ./main.go
 
 .PHONY: local_run
 local_run:

cmd/duplicate_versions.go

@@ -27,7 +27,7 @@ var Versions = &cobra.Command{
 		cfg := config.NewConfig(envFile)
 
 		bbService := services.NewBackBlaze(cfg.BbId, cfg.BbKey)
-		if err := bbService.ListDuplicateVersions(ctx); err != nil {
+		if err := bbService.ListDuplicateVersions(ctx, cancel); err != nil {
 			log.Fatalln("bbService.ListDuplicateVersions()", err)
 		}
 	},

internal/services/backblaze.go

@@ -308,7 +308,7 @@ type duplicate struct {
 	count  int
 }
 
-func (b *BackBalze) ListDuplicateVersions(ctx context.Context) error {
+func (b *BackBalze) ListDuplicateVersions(ctx context.Context, cancel context.CancelFunc) error {
 	b2Client, err := b2.NewClient(ctx, b.bbID, b.bbKey)
 	if err != nil {
 		return fmt.Errorf("b2.NewClient %w", err)
@@ -320,44 +320,63 @@ func (b *BackBalze) ListDuplicateVersions(ctx context.Context) error {
 		return fmt.Errorf("b2Client.Bucket %w", err)
 	}
 
+	wg := sync.WaitGroup{}
 	dups := make([]duplicate, 0)
 
 	log.Println("len(buckets)", len(buckets))
+	sm := semaphore.NewWeighted(int64(b.maxWorkers))
+	wg.Add(len(buckets))
 	for _, bc := range buckets {
-		files := make(map[string]int, 0)
+		if err := sm.Acquire(ctx, 1); err != nil {
-
+			return fmt.Errorf("sm.Acquire %w", err)
-		bucketIter := bc.List(ctx, b2.ListHidden())
-		if bucketIter == nil {
-			return fmt.Errorf("bucket list cannot be nil")
 		}
 
-		for {
+		go func(bc *b2.Bucket) {
-			if !bucketIter.Next() {
+			defer sm.Release(1)
-				if bucketIter.Err() != nil {
+			defer wg.Done()
-					return fmt.Errorf("bucketIter err %w", bucketIter.Err())
+			files := make(map[string]int, 0)
+
+			bucketIter := bc.List(ctx, b2.ListHidden())
+			if bucketIter == nil {
+				log.Println("bucket list cannot be nil")
+				return
+			}
+
+			for {
+				if !bucketIter.Next() {
+					if bucketIter.Err() != nil {
+						log.Println("bucketIter err %w", bucketIter.Err())
+						return
+					}
+					break
 				}
-				break
+				if bucketIter.Object() == nil {
+					log.Println("bucketIter Object is nil")
+					continue
+				}
+				files[bucketIter.Object().Name()]++
 			}
-			if bucketIter.Object() == nil {
-				log.Println("bucketIter Object is nil")
-				continue
-			}
-			files[bucketIter.Object().Name()]++
-		}
 
-		// Search duplicates
+			// Search duplicates
-		for file, count := range files {
+			for file, count := range files {
-			if count > 1 {
+				if count > 1 {
-				dups = append(dups, duplicate{
+					dups = append(dups, duplicate{
-					bucket: bc.Name(),
+						bucket: bc.Name(),
-					file:   file,
+						file:   file,
-					count:  count,
+						count:  count,
-				})
+					})
+				}
 			}
-		}
+		}(bc)
 	}
+	wg.Wait()
+
 	if len(dups) > 0 {
-		return fmt.Errorf("found duplicates: %+v", dups)
+		var builder strings.Builder
+		for _, dup := range dups {
+			builder.WriteString(fmt.Sprintf("%+v\n", dup))
+		}
+		return fmt.Errorf("found duplicates: %s", builder.String())
 	}
 	return nil
 }