diff --git a/cmd/check.go b/cmd/check.go index 2e4ac8f..165e586 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -3,9 +3,9 @@ package cmd import ( "bytes" "context" - "errors" "fmt" "log" + "math" "net/smtp" "os" "strings" @@ -72,20 +72,27 @@ var Check = &cobra.Command{ logger.Info("start check") defer logger.Info("end check") - msgChan := make(chan backblaze.B2Local) - go b.CompareConcurrent(ctx, backupDir, bucketName, msgChan) + localChan := make(chan backblaze.B2Local) + b2Chan := make(chan backblaze.B2Local) + doneChan := make(chan interface{}, 1) + go b.CompareConcurrent(ctx, backupDir, bucketName, localChan, b2Chan, doneChan) reportBuilder := strings.Builder{} countLocal := 0 + countB2 := 0 countOK := 0 reportBuilder.WriteString(fmt.Sprintf("Local files within `%s` path already in `%s` bucket:\n", backupDir, bucketName)) cloudBuilder := strings.Builder{} - cloudBuilder.WriteString(fmt.Sprintf("List of B2 files within `%s` bucket not found in local path `%s`\n", bucketName, backupDir)) + cloudBuilder.WriteString(fmt.Sprintf("B2 files within `%s` bucket\n", bucketName)) + cloudBuilder.WriteString("| B2 File Name | Size | ModTime | Status |\n") + cloudBuilder.WriteString("|----------------------------------------------------------|------------------|--------------------------------|------------------------------------------\n") countNotInLocal := 0 localBuilder := strings.Builder{} - localBuilder.WriteString(fmt.Sprintf("List of local files in `%s` not found in B2 bucket `%s`\n", backupDir, bucketName)) + localBuilder.WriteString(fmt.Sprintf("Local files in `%s`\n", backupDir)) + localBuilder.WriteString("| Local File Name | Size | ModTime | Status |\n") + localBuilder.WriteString("|----------------------------------------------------------|------------------|--------------------------------|------------------------------------------\n") countNotInCloud := 0 loop: @@ -93,55 +100,73 @@ var Check = &cobra.Command{ select { case <-ctx.Done(): // Handle the timeout or cancellation - // Release any resources here if ctx.Err() == context.DeadlineExceeded { logger.Error("Operation timed out") } else if ctx.Err() == context.Canceled { logger.Error("Operation canceled") } break loop - case msg := <-msgChan: - if msg.Err == nil && msg.File == "" { - countLocal = msg.LocalCount - break loop - } - if errors.Is(msg.Err, backblaze.ErrCloudNotInLocal) { - logger.Debug(msg.File + ": B2 file not found in local") - cloudBuilder.WriteString(msg.File + "\n") - countNotInLocal++ + case <-doneChan: + logger.Debugln("done chan") + break loop + case localMsg, ok := <-localChan: + if !ok { continue } - if errors.Is(msg.Err, backblaze.ErrLocalNotInCloud) { - logger.Debug(msg.File + ": local file not found in B2") - localBuilder.WriteString(msg.File + "\n") - countNotInCloud++ + countLocal++ + logger.Debugln("localChan", localMsg.File) + writeTableRow(&localBuilder, &countNotInCloud, localMsg.File, localMsg.Err) + case b2Msg, ok := <-b2Chan: + if !ok { continue } - reportBuilder.WriteString(msg.File + " OK" + "\n") - countOK++ + countB2++ + logger.Debugln("cloudBuilder", b2Msg.File) + writeTableRow(&cloudBuilder, &countNotInLocal, b2Msg.File, b2Msg.Err) } } - if countNotInLocal > 0 { - reportBuilder.WriteString("\n") - reportBuilder.WriteString(cloudBuilder.String()) - } - if countNotInCloud > 0 { - reportBuilder.WriteString("\n") - reportBuilder.WriteString(localBuilder.String()) - } + reportBuilder.WriteString("\n") + reportBuilder.WriteString(cloudBuilder.String()) + reportBuilder.WriteString("\n") + reportBuilder.WriteString(localBuilder.String()) if err := mailSrv.SendOK(email.EmailWithAttachments{ - To: cfg.MailTo, - Bucket: bucketName, - BackupDir: backupDir, - CountNotInLocal: countNotInLocal, - CountNotInCloud: countNotInCloud, - CountOK: countOK, - CountLocal: countLocal, - Attachments: []email.EmailAttachment{{File: bytes.NewReader([]byte(reportBuilder.String())), Title: fmt.Sprintf("%s-check-report.txt", bucketName)}}, + To: cfg.MailTo, + Bucket: bucketName, + BackupDir: backupDir, + Count: map[string]int{ + "count_ok": countOK, + "count_local": countLocal, + "count_b2": countB2, + "count_not_in_local": countNotInLocal, + "count_not_in_cloud": countNotInCloud, + }, + Attachments: []email.EmailAttachment{{File: bytes.NewReader([]byte(reportBuilder.String())), Title: fmt.Sprintf("%s-check-report.txt", bucketName)}}, }); err != nil { panic(fmt.Errorf("error while send email: %w", err)) } }, } + +// New function to write a table row +func writeTableRow(builder *strings.Builder, count *int, file backblaze.File, err error) { + status := "OK" + if err != nil { + status = err.Error() + (*count)++ + } + + builder.WriteString(fmt.Sprintf("| %-56s | %-16s | %-30s | %-40s |\n", file.Path, prettyByteSize(file.Size), file.ModTime.Format("2006-01-02 15:04:05.000"), status)) +} + +func prettyByteSize(b int) string { + bf := float64(b) + for _, unit := range []string{"", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"} { + if math.Abs(bf) < 1024.0 { + return fmt.Sprintf("%3.1f%sB", bf, unit) + } + bf /= 1024.0 + } + return fmt.Sprintf("%.1fYiB", bf) +} diff --git a/cmd/check_test.go b/cmd/check_test.go new file mode 100644 index 0000000..f998889 --- /dev/null +++ b/cmd/check_test.go @@ -0,0 +1,11 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_prettyByteSize(t *testing.T) { + require.Equal(t, prettyByteSize(1505099776), "1.4GiB") +} diff --git a/internal/services/backblaze/check.go b/internal/services/backblaze/check.go index 6aa861e..d0f866d 100644 --- a/internal/services/backblaze/check.go +++ b/internal/services/backblaze/check.go @@ -8,13 +8,21 @@ import ( "path" "path/filepath" "sync" + "time" "github.com/kurin/blazer/b2" ) +type File struct { + Path string + Size int // file size in bytes + ModTime time.Time + IsUploaded *bool +} + // localFiles lists the local files in the given backup directory and sends them to a channel. // It closes the channel after all files are listed. -func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- string) error { +func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- File) error { defer close(fileChan) // Walk the directory and send files to the channel err := filepath.WalkDir(backupDir, func(path string, d fs.DirEntry, err error) error { @@ -22,12 +30,16 @@ func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- string) error { return err } if !d.IsDir() { - fileChan <- filepath.Base(path) + i, err := d.Info() + if err != nil { + return fmt.Errorf("d.Info: %w", err) + } + fileChan <- File{Path: filepath.Base(path), Size: int(i.Size()), ModTime: i.ModTime()} } return nil }) if err != nil { - return fmt.Errorf("error walking the directory: %v", err) + return fmt.Errorf("error walking the directory: %w", err) } return nil @@ -35,7 +47,7 @@ func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- string) error { // b2BucketFiles lists the files in the given B2 bucket and sends them to a channel. // It closes the channel after all files are listed. -func (b *BackBlaze) b2BucketFiles(ctx context.Context, bucketName string, fileChan chan<- string) error { +func (b *BackBlaze) b2BucketFiles(ctx context.Context, bucketName string, fileChan chan<- File) error { bucket, err := b.b2Client.Bucket(ctx, bucketName) defer close(fileChan) if err != nil { @@ -58,32 +70,35 @@ func (b *BackBlaze) b2BucketFiles(ctx context.Context, bucketName string, fileCh return errors.New("bucketIter Object is nil") } - // Retrieve just filename - b.logger.Debugln("bucket file: ", path.Base(bucketIter.Object().Name())) - fileChan <- path.Base(bucketIter.Object().Name()) + fileName := path.Base(bucketIter.Object().Name()) + attrs, err := bucketIter.Object().Attrs(ctx) + if err != nil { + return fmt.Errorf("bucketIter.Object().Attrs %s err %w", fileName, bucketIter.Err()) + } + isUploaded := attrs.Status != b2.Uploaded + fileChan <- File{Path: path.Base(fileName), Size: int(attrs.Size), IsUploaded: &isUploaded, ModTime: attrs.UploadTimestamp} } return nil } var ErrLocalNotInCloud error = errors.New("exists locally but not in the cloud") -var ErrCloudNotInLocal error = errors.New("exists on cloud but not locally") +var ErrCloudNotInLocal error = errors.New("exists on B2 but not locally") type B2Local struct { - File string - Err error - LocalCount int + File File + Err error } // CompareConcurrent concurrently fetches the list of local files and cloud files, // then compares them to ensure all local files exist in the cloud. // Errors are sent to a provided error channel. The function will panic if an error occurs while listing files. -func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName string, msgChan chan<- B2Local) { +func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName string, localChan chan<- B2Local, b2Chan chan<- B2Local, doneChan chan<- interface{}) { var wg sync.WaitGroup - localFiles := make(map[string]int) - cloudFiles := make(map[string]int) - localFileChan := make(chan string) - b2FileChan := make(chan string) + localFiles := make(map[string]File) + cloudFiles := make(map[string]File) + localFileChan := make(chan File) + b2FileChan := make(chan File) // Local listing wg.Add(1) @@ -93,11 +108,11 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName go func() { defer wg.Done() for f := range localFileChan { - if _, ok := localFiles[f]; ok { - panic(fmt.Errorf("local file already exists in map: %s", f)) + if _, ok := localFiles[f.Path]; ok { + panic(fmt.Errorf("local file already exists in map: %s", f.Path)) } - b.logger.Debugln("local file ", f) - localFiles[f]++ + b.logger.Debugf("local file %+v\n", f) + localFiles[f.Path] = f } }() @@ -114,11 +129,11 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName go func() { defer wg.Done() for f := range b2FileChan { - if _, ok := cloudFiles[f]; ok { - panic(fmt.Errorf("cloud file already exists in map: %s", f)) + if _, ok := cloudFiles[f.Path]; ok { + panic(fmt.Errorf(`cloud file already exists in map: %s\n you should run 'backblazebackup cleanup --bucket "%s"'`, f.Path, b.bucketName)) } - b.logger.Debugln("B2 file ", f) - cloudFiles[f]++ + b.logger.Debugf("B2 file %+v\n", f) + cloudFiles[f.Path] = f } }() if err := b.b2BucketFiles(ctx, bucketName, b2FileChan); err != nil { @@ -133,27 +148,29 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName wg.Add(2) go func() { defer wg.Done() - for localFile := range localFiles { - if _, exists := cloudFiles[localFile]; !exists { - msgChan <- B2Local{File: localFile, Err: ErrLocalNotInCloud} + for path, localFile := range localFiles { + if _, exists := cloudFiles[path]; !exists { + localChan <- B2Local{File: localFile, Err: ErrLocalNotInCloud} continue } - msgChan <- B2Local{File: localFile, Err: nil} + b.logger.Debugf("localFile %+v\n", localFile) + localChan <- B2Local{File: localFile, Err: nil} } }() // Now check cloud files that are not in local go func() { defer wg.Done() - for cloudFile := range cloudFiles { - b.logger.Debugln("cloudFile ", cloudFile) - if _, exists := localFiles[cloudFile]; !exists { - msgChan <- B2Local{File: cloudFile, Err: ErrCloudNotInLocal} + for path, cloudFile := range cloudFiles { + if _, exists := localFiles[path]; !exists { + b2Chan <- B2Local{File: cloudFile, Err: ErrCloudNotInLocal} + continue } + b2Chan <- B2Local{File: cloudFile, Err: nil} } }() - wg.Wait() - msgChan <- B2Local{Err: nil, LocalCount: len(localFiles)} - close(msgChan) + close(localChan) + close(b2Chan) + doneChan <- nil } diff --git a/internal/services/email/email.go b/internal/services/email/email.go index 9d6aff3..dbce53b 100644 --- a/internal/services/email/email.go +++ b/internal/services/email/email.go @@ -29,14 +29,11 @@ type EmailService struct { } type EmailWithAttachments struct { - To string - Bucket string - BackupDir string - CountNotInLocal int - CountOK int - CountNotInCloud int - CountLocal int - Attachments []EmailAttachment + To string + Bucket string + BackupDir string + Count map[string]int + Attachments []EmailAttachment } type EmailAttachment struct { @@ -75,11 +72,9 @@ func NewMailService(config MailServiceConfig) *EmailService { func (e *EmailService) SendOK(emailData EmailWithAttachments) error { template := strings.Replace(htmlTemplate, "{{bucket}}", emailData.Bucket, -1) template = strings.Replace(template, "{{local_backup_path}}", emailData.BackupDir, -1) - template = strings.Replace(template, "{{count_ErrCloudNotInLocal}}", fmt.Sprint(emailData.CountNotInLocal), -1) - template = strings.Replace(template, "{{count_ErrLocalNotInCloud}}", fmt.Sprint(emailData.CountNotInCloud), -1) - template = strings.Replace(template, "{{count_ok}}", fmt.Sprint(emailData.CountOK), -1) - template = strings.Replace(template, "{{count_local}}", fmt.Sprint(emailData.CountLocal), -1) - + for k, v := range emailData.Count { + template = strings.Replace(template, "{{"+k+"}}", fmt.Sprint(v), -1) + } msg, err := newMessage(e.from, emailData.To, subjectReport). withAttachments(template, emailData.Attachments) diff --git a/internal/services/email/template.go b/internal/services/email/template.go index 7a4e941..913774f 100644 --- a/internal/services/email/template.go +++ b/internal/services/email/template.go @@ -11,11 +11,12 @@ const htmlTemplate = `

Bucket Report


Local Backup Path: {{local_backup_path}}

-

Local files: {{count_local}}

Bucket Name: {{bucket}}

+

Local files: {{count_local}}

+

B2 files: {{count_b2}}

OK local in B2: {{count_ok}}

-

B2 files not in local: {{count_ErrCloudNotInLocal}}

-

Local files not in B2: {{count_ErrLocalNotInCloud}}

+

B2 files not in local: {{count_not_in_local}}

+

Local files not in B2: {{count_not_in_cloud}}


This is an automated report, please do not reply to this email.