From f43fae8c3de49a7acd7acc5353ff56ecfc0a69eb Mon Sep 17 00:00:00 2001 From: Urko Date: Mon, 28 Aug 2023 11:01:57 +0200 Subject: [PATCH] feat: improve display report --- cmd/check.go | 69 +++++++++++++++++----------- internal/services/backblaze/check.go | 25 +++++----- internal/services/email/email.go | 16 ++++--- internal/services/email/template.go | 6 ++- 4 files changed, 69 insertions(+), 47 deletions(-) diff --git a/cmd/check.go b/cmd/check.go index 93b3615..ba5d89e 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -8,6 +8,7 @@ import ( "log" "net/smtp" "os" + "strings" "time" "gitea.urkob.com/urko/backblaze-backup/internal/services/backblaze" @@ -71,16 +72,20 @@ var Check = &cobra.Command{ logger.Info("start check") defer logger.Info("end check") - errs := make(chan backblaze.B2LocalErr) - go b.CompareConcurrent(ctx, backupDir, bucketName, errs) + msgChan := make(chan backblaze.B2Local) + go b.CompareConcurrent(ctx, backupDir, bucketName, msgChan) - // Create two buffers for two kinds of errors - cloudNotInLocalBuffer := new(bytes.Buffer) - cloudNotInLocalBuffer.WriteString(fmt.Sprintf("List of B2 files within %s bucket not found in local path %s \n", bucketName, backupDir)) + reportBuilder := strings.Builder{} + countLocalFiles := 0 + countOK := 0 + reportBuilder.WriteString(fmt.Sprintf("Local files within `%s` path already in `%s` bucket:\n", backupDir, bucketName)) + + cloudBuilder := strings.Builder{} + cloudBuilder.WriteString(fmt.Sprintf("List of B2 files within `%s` bucket not found in local path `%s`\n", bucketName, backupDir)) countNotInLocalBuffer := 0 - localNotInCloudBuffer := new(bytes.Buffer) - localNotInCloudBuffer.WriteString(fmt.Sprintf("List of local files in %s not found in B2 bucket %s \n", backupDir, bucketName)) + localBuilder := strings.Builder{} + localBuilder.WriteString(fmt.Sprintf("List of local files in `%s` not found in B2 bucket `%s`\n", backupDir, bucketName)) countNotInCloudBuffer := 0 loop: @@ -95,38 +100,46 @@ var Check = &cobra.Command{ logger.Error("Operation canceled") } break loop - case err := <-errs: - if errors.Is(err.Err, backblaze.ErrCloudNotInLocal) { - logger.Debug(err.File + ": B2 file not found in local") - cloudNotInLocalBuffer.WriteString(err.File + "\n") - countNotInLocalBuffer++ - } - if errors.Is(err.Err, backblaze.ErrLocalNotInCloud) { - logger.Debug(err.File + ": local file not found in B2") - localNotInCloudBuffer.WriteString(err.File + "\n") - countNotInCloudBuffer++ - } - if err.Err == nil { + case msg := <-msgChan: + if msg.Err == nil && msg.File == "" { + countLocalFiles = msg.LocalCount break loop } + if errors.Is(msg.Err, backblaze.ErrCloudNotInLocal) { + logger.Debug(msg.File + ": B2 file not found in local") + cloudBuilder.WriteString(msg.File + "\n") + countNotInLocalBuffer++ + continue + } + if errors.Is(msg.Err, backblaze.ErrLocalNotInCloud) { + logger.Debug(msg.File + ": local file not found in B2") + localBuilder.WriteString(msg.File + "\n") + countNotInCloudBuffer++ + continue + } + reportBuilder.WriteString(msg.File + " OK" + "\n") + countOK++ } } - var attachments []email.EmailAttachment if countNotInCloudBuffer > 0 { - attachments = append(attachments, email.EmailAttachment{File: cloudNotInLocalBuffer, Title: "B2-Files-Not-In-Local.txt"}) + reportBuilder.WriteString("\n") + reportBuilder.WriteString(cloudBuilder.String()) } if countNotInLocalBuffer > 0 { - attachments = append(attachments, email.EmailAttachment{File: localNotInCloudBuffer, Title: "Local-Files-Not-In-B2.txt"}) + reportBuilder.WriteString("\n") + reportBuilder.WriteString(localBuilder.String()) } if err := mailSrv.SendOK(email.EmailWithAttachments{ - To: cfg.MailTo, - Bucket: bucketName, - BackupDir: backupDir, - CountLocal: countNotInLocalBuffer, - CountCloud: countNotInCloudBuffer, - Attachments: attachments, + To: cfg.MailTo, + Bucket: bucketName, + BackupDir: backupDir, + CountLocal: countNotInLocalBuffer, + CountCloud: countNotInCloudBuffer, + CountOK: countOK, + CountLocalFiles: countLocalFiles, + Attachments: []email.EmailAttachment{{File: bytes.NewReader([]byte(reportBuilder.String())), Title: fmt.Sprintf("%s-check-report.txt", bucketName)}}, }); err != nil { panic(fmt.Errorf("error while send email: %w", err)) } diff --git a/internal/services/backblaze/check.go b/internal/services/backblaze/check.go index 4a7920f..1e595ab 100644 --- a/internal/services/backblaze/check.go +++ b/internal/services/backblaze/check.go @@ -66,15 +66,16 @@ func (b *BackBlaze) b2BucketFiles(ctx context.Context, bucketName string, fileCh var ErrLocalNotInCloud error = errors.New("exists locally but not in the cloud") var ErrCloudNotInLocal error = errors.New("exists on cloud but not locally") -type B2LocalErr struct { - File string - Err error +type B2Local struct { + File string + Err error + LocalCount int } // CompareConcurrent concurrently fetches the list of local files and cloud files, // then compares them to ensure all local files exist in the cloud. // Errors are sent to a provided error channel. The function will panic if an error occurs while listing files. -func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName string, errors chan<- B2LocalErr) { +func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName string, msgChan chan<- B2Local) { var wg sync.WaitGroup localFiles := make(map[string]int) cloudFiles := make(map[string]int) @@ -92,7 +93,7 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName if _, ok := localFiles[f]; ok { panic(fmt.Errorf("local file already exists in map: %s", f)) } - b.logger.Debug("local file", f) + b.logger.Debug("local file ", f) localFiles[f]++ } }() @@ -113,7 +114,7 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName if _, ok := cloudFiles[f]; ok { panic(fmt.Errorf("cloud file already exists in map: %s", f)) } - b.logger.Debug("B2 file", f) + b.logger.Debug("B2 file ", f) cloudFiles[f]++ } }() @@ -125,14 +126,16 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName // Wait for both to complete wg.Wait() - // Now check local files that are not presesnt in cloud + // Now check local files that are not present in cloud wg.Add(2) go func() { defer wg.Done() for localFile := range localFiles { if _, exists := cloudFiles[localFile]; !exists { - errors <- B2LocalErr{File: localFile, Err: ErrLocalNotInCloud} + msgChan <- B2Local{File: localFile, Err: ErrLocalNotInCloud} + continue } + msgChan <- B2Local{File: localFile, Err: nil} } }() @@ -141,12 +144,12 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName defer wg.Done() for cloudFile := range cloudFiles { if _, exists := localFiles[cloudFile]; !exists { - errors <- B2LocalErr{File: cloudFile, Err: ErrCloudNotInLocal} + msgChan <- B2Local{File: cloudFile, Err: ErrCloudNotInLocal} } } }() wg.Wait() - errors <- B2LocalErr{Err: nil} - close(errors) + msgChan <- B2Local{Err: nil, LocalCount: len(localFiles)} + close(msgChan) } diff --git a/internal/services/email/email.go b/internal/services/email/email.go index d88c962..1d7e2b3 100644 --- a/internal/services/email/email.go +++ b/internal/services/email/email.go @@ -29,12 +29,14 @@ type EmailService struct { } type EmailWithAttachments struct { - To string - Bucket string - BackupDir string - CountLocal int - CountCloud int - Attachments []EmailAttachment + To string + Bucket string + BackupDir string + CountLocal int + CountOK int + CountCloud int + CountLocalFiles int + Attachments []EmailAttachment } type EmailAttachment struct { @@ -75,6 +77,8 @@ func (e *EmailService) SendOK(emailData EmailWithAttachments) error { template = strings.Replace(template, "{{local_backup_path}}", emailData.BackupDir, -1) template = strings.Replace(template, "{{count_ErrCloudNotInLocal}}", fmt.Sprint(emailData.CountCloud), -1) template = strings.Replace(template, "{{count_ErrLocalNotInCloud}}", fmt.Sprint(emailData.CountLocal), -1) + template = strings.Replace(template, "{{count_ok}}", fmt.Sprint(emailData.CountOK), -1) + template = strings.Replace(template, "{{count_local}}", fmt.Sprint(emailData.CountLocalFiles), -1) msg, err := newMessage(e.from, emailData.To, subjectReport). withAttachments(template, emailData.Attachments) diff --git a/internal/services/email/template.go b/internal/services/email/template.go index 6c90f1e..7a4e941 100644 --- a/internal/services/email/template.go +++ b/internal/services/email/template.go @@ -11,9 +11,11 @@ const htmlTemplate = `

Bucket Report

Local Backup Path: {{local_backup_path}}

+

Local files: {{count_local}}

Bucket Name: {{bucket}}

-

Count of ErrCloudNotInLocal: {{count_ErrCloudNotInLocal}}

-

Count of ErrLocalNotInCloud: {{count_ErrLocalNotInCloud}}

+

OK local in B2: {{count_ok}}

+

B2 files not in local: {{count_ErrCloudNotInLocal}}

+

Local files not in B2: {{count_ErrLocalNotInCloud}}

This is an automated report, please do not reply to this email.