package backblaze import ( "context" "errors" "fmt" "io" "io/fs" "log" "os" "path/filepath" "strings" "sync" "github.com/kurin/blazer/b2" ) const writers = 4 func (b *BackBlaze) Sync(ctx context.Context) error { if b.bucketName == "" && (b.filePath == "" || b.dir == "") { return fmt.Errorf("bucket name is %v | filePath is %v | dir is %v", b.bucketName, b.filePath, b.dir) } if b.filePath != "" && b.dir != "" { return errors.New("you must select just 1 option, dir or file") } bc, err := b.b2Client.Bucket(ctx, b.bucketName) if err != nil { return fmt.Errorf("b2Client.Bucket %w", err) } if bc == nil { return fmt.Errorf("bucket doesn't exist %s", b.bucketName) } log.Println("bucket found:", bc.Name()) if b.filePath != "" { log.Println("file:", b.filePath) if err := b.copyFile(ctx, bc, b.filePath); err != nil { return fmt.Errorf("copyFile %w", err) } return nil } if b.dir != "" { oldFiles, err := b.bucketFiles(ctx, bc) if err != nil { return fmt.Errorf("bucketFiles %w", err) } b.logger.Debug(strings.Repeat("*", 40)) b.logger.Debug("oldFiles to clean:\n\t\t" + strings.Join(oldFiles, "\n\t\t")) b.logger.Debug(strings.Repeat("*", 40)) fileChan := make(chan string) var wg sync.WaitGroup for i := 0; i < b.maxWorkers; i++ { wg.Add(1) go func() { defer wg.Done() for src := range fileChan { if err := b.copyFile(ctx, bc, src); err != nil { b.logger.Errorf("error copying file %s: %v\n", src, err) continue } } }() } // Walk the directory and send files to the channel for uploading err = filepath.WalkDir(b.dir, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if !d.IsDir() { fileChan <- path } return nil }) if err != nil { return fmt.Errorf("error walking the directory: %v", err) } // Close the channel (no more files to send) close(fileChan) wg.Wait() // Cleanup old files after backup is completed if err := b.cleanBucket(ctx, bc, oldFiles); err != nil { return fmt.Errorf("cleanBucket %w", err) } } b.logger.Info("copied successfully") return nil } func (b *BackBlaze) copyFile(ctx context.Context, bucket *b2.Bucket, src string) error { f, err := os.Open(src) if err != nil { return err } defer f.Close() fi, err := f.Stat() if err != nil { return err } w := bucket.Object(fi.Name()).NewWriter(ctx) w.ConcurrentUploads = writers w.UseFileBuffer = true b.logger.Info("start copying", fi.Name()) if _, err := io.Copy(w, f); err != nil { w.Close() return err } if err := w.Close(); err != nil { return err } b.logger.Info("end copying", fi.Name()) return nil } func (b *BackBlaze) cleanBucket(ctx context.Context, bucket *b2.Bucket, files []string) error { var errorBuilder strings.Builder for _, v := range files { obj := bucket.Object(v) if obj == nil { b.logger.Error("bucket.Object is nil", v) continue } if err := obj.Delete(ctx); err != nil { errorBuilder.WriteString(fmt.Errorf("error deleting %s : %w", v, err).Error()) errorBuilder.WriteString("; ") } } if errorBuilder.Len() > 0 { return errors.New(errorBuilder.String()) } return nil } func (b *BackBlaze) bucketFiles(ctx context.Context, bucket *b2.Bucket) ([]string, error) { bucketIter := bucket.List(ctx) if bucketIter == nil { return nil, fmt.Errorf("bucket list cannot be nil") } var files []string for { if !bucketIter.Next() { if bucketIter.Err() != nil { return nil, fmt.Errorf("bucketIter err %w", bucketIter.Err()) } break } if bucketIter.Object() == nil { b.logger.Error("bucketIter Object is nil") continue } files = append(files, bucketIter.Object().Name()) } return files, nil }