feat: improve report add columns to display modtime and size

cmd/check.go

@@ -3,9 +3,9 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"log"
+	"math"
 	"net/smtp"
 	"os"
 	"strings"
@@ -72,20 +72,27 @@ var Check = &cobra.Command{
 
 		logger.Info("start check")
 		defer logger.Info("end check")
-		msgChan := make(chan backblaze.B2Local)
+		localChan := make(chan backblaze.B2Local)
-		go b.CompareConcurrent(ctx, backupDir, bucketName, msgChan)
+		b2Chan := make(chan backblaze.B2Local)
+		doneChan := make(chan interface{}, 1)
+		go b.CompareConcurrent(ctx, backupDir, bucketName, localChan, b2Chan, doneChan)
 
 		reportBuilder := strings.Builder{}
 		countLocal := 0
+		countB2 := 0
 		countOK := 0
 		reportBuilder.WriteString(fmt.Sprintf("Local files within `%s` path already in `%s` bucket:\n", backupDir, bucketName))
 
 		cloudBuilder := strings.Builder{}
-		cloudBuilder.WriteString(fmt.Sprintf("List of B2 files within `%s` bucket not found in local path `%s`\n", bucketName, backupDir))
+		cloudBuilder.WriteString(fmt.Sprintf("B2 files within `%s` bucket\n", bucketName))
+		cloudBuilder.WriteString("| B2 File Name                                             | Size             | ModTime                        | Status                                   |\n")
+		cloudBuilder.WriteString("|----------------------------------------------------------|------------------|--------------------------------|------------------------------------------\n")
 		countNotInLocal := 0
 
 		localBuilder := strings.Builder{}
-		localBuilder.WriteString(fmt.Sprintf("List of local files in `%s` not found in B2 bucket `%s`\n", backupDir, bucketName))
+		localBuilder.WriteString(fmt.Sprintf("Local files in `%s`\n", backupDir))
+		localBuilder.WriteString("| Local File Name                                          | Size             | ModTime                        | Status                                   |\n")
+		localBuilder.WriteString("|----------------------------------------------------------|------------------|--------------------------------|------------------------------------------\n")
 		countNotInCloud := 0
 
 	loop:
@@ -93,55 +100,73 @@ var Check = &cobra.Command{
 			select {
 			case <-ctx.Done():
 				// Handle the timeout or cancellation
-				// Release any resources here
 				if ctx.Err() == context.DeadlineExceeded {
 					logger.Error("Operation timed out")
 				} else if ctx.Err() == context.Canceled {
 					logger.Error("Operation canceled")
 				}
 				break loop
-			case msg := <-msgChan:
+			case <-doneChan:
-				if msg.Err == nil && msg.File == "" {
+				logger.Debugln("done chan")
-					countLocal = msg.LocalCount
 				break loop
-				}
+			case localMsg, ok := <-localChan:
-				if errors.Is(msg.Err, backblaze.ErrCloudNotInLocal) {
+				if !ok {
-					logger.Debug(msg.File + ": B2 file not found in local")
-					cloudBuilder.WriteString(msg.File + "\n")
-					countNotInLocal++
 					continue
 				}
-				if errors.Is(msg.Err, backblaze.ErrLocalNotInCloud) {
+				countLocal++
-					logger.Debug(msg.File + ": local file not found in B2")
+				logger.Debugln("localChan", localMsg.File)
-					localBuilder.WriteString(msg.File + "\n")
+				writeTableRow(&localBuilder, &countNotInCloud, localMsg.File, localMsg.Err)
-					countNotInCloud++
+			case b2Msg, ok := <-b2Chan:
+				if !ok {
 					continue
 				}
-				reportBuilder.WriteString(msg.File + " OK" + "\n")
+				countB2++
-				countOK++
+				logger.Debugln("cloudBuilder", b2Msg.File)
+				writeTableRow(&cloudBuilder, &countNotInLocal, b2Msg.File, b2Msg.Err)
 			}
 		}
 
-		if countNotInLocal > 0 {
 		reportBuilder.WriteString("\n")
 		reportBuilder.WriteString(cloudBuilder.String())
-		}
-		if countNotInCloud > 0 {
 		reportBuilder.WriteString("\n")
 		reportBuilder.WriteString(localBuilder.String())
-		}
 
 		if err := mailSrv.SendOK(email.EmailWithAttachments{
 			To:        cfg.MailTo,
 			Bucket:    bucketName,
 			BackupDir: backupDir,
-			CountNotInLocal: countNotInLocal,
+			Count: map[string]int{
-			CountNotInCloud: countNotInCloud,
+				"count_ok":           countOK,
-			CountOK:         countOK,
+				"count_local":        countLocal,
-			CountLocal:      countLocal,
+				"count_b2":           countB2,
+				"count_not_in_local": countNotInLocal,
+				"count_not_in_cloud": countNotInCloud,
+			},
 			Attachments: []email.EmailAttachment{{File: bytes.NewReader([]byte(reportBuilder.String())), Title: fmt.Sprintf("%s-check-report.txt", bucketName)}},
 		}); err != nil {
 			panic(fmt.Errorf("error while send email: %w", err))
 		}
 	},
 }
+
+// New function to write a table row
+func writeTableRow(builder *strings.Builder, count *int, file backblaze.File, err error) {
+	status := "OK"
+	if err != nil {
+		status = err.Error()
+		(*count)++
+	}
+
+	builder.WriteString(fmt.Sprintf("| %-56s | %-16s | %-30s | %-40s |\n", file.Path, prettyByteSize(file.Size), file.ModTime.Format("2006-01-02 15:04:05.000"), status))
+}
+
+func prettyByteSize(b int) string {
+	bf := float64(b)
+	for _, unit := range []string{"", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"} {
+		if math.Abs(bf) < 1024.0 {
+			return fmt.Sprintf("%3.1f%sB", bf, unit)
+		}
+		bf /= 1024.0
+	}
+	return fmt.Sprintf("%.1fYiB", bf)
+}

cmd/check_test.go

@@ -0,0 +1,11 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_prettyByteSize(t *testing.T) {
+	require.Equal(t, prettyByteSize(1505099776), "1.4GiB")
+}

internal/services/backblaze/check.go

@@ -8,13 +8,21 @@ import (
 	"path"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"github.com/kurin/blazer/b2"
 )
 
+type File struct {
+	Path       string
+	Size       int // file size in bytes
+	ModTime    time.Time
+	IsUploaded *bool
+}
+
 // localFiles lists the local files in the given backup directory and sends them to a channel.
 // It closes the channel after all files are listed.
-func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- string) error {
+func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- File) error {
 	defer close(fileChan)
 	// Walk the directory and send files to the channel
 	err := filepath.WalkDir(backupDir, func(path string, d fs.DirEntry, err error) error {
@@ -22,12 +30,16 @@ func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- string) error {
 			return err
 		}
 		if !d.IsDir() {
-			fileChan <- filepath.Base(path)
+			i, err := d.Info()
+			if err != nil {
+				return fmt.Errorf("d.Info: %w", err)
+			}
+			fileChan <- File{Path: filepath.Base(path), Size: int(i.Size()), ModTime: i.ModTime()}
 		}
 		return nil
 	})
 	if err != nil {
-		return fmt.Errorf("error walking the directory: %v", err)
+		return fmt.Errorf("error walking the directory: %w", err)
 	}
 
 	return nil
@@ -35,7 +47,7 @@ func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- string) error {
 
 // b2BucketFiles lists the files in the given B2 bucket and sends them to a channel.
 // It closes the channel after all files are listed.
-func (b *BackBlaze) b2BucketFiles(ctx context.Context, bucketName string, fileChan chan<- string) error {
+func (b *BackBlaze) b2BucketFiles(ctx context.Context, bucketName string, fileChan chan<- File) error {
 	bucket, err := b.b2Client.Bucket(ctx, bucketName)
 	defer close(fileChan)
 	if err != nil {
@@ -58,32 +70,35 @@ func (b *BackBlaze) b2BucketFiles(ctx context.Context, bucketName string, fileCh
 			return errors.New("bucketIter Object is nil")
 		}
 
-		// Retrieve just filename
+		fileName := path.Base(bucketIter.Object().Name())
-		b.logger.Debugln("bucket file: ", path.Base(bucketIter.Object().Name()))
+		attrs, err := bucketIter.Object().Attrs(ctx)
-		fileChan <- path.Base(bucketIter.Object().Name())
+		if err != nil {
+			return fmt.Errorf("bucketIter.Object().Attrs %s err %w", fileName, bucketIter.Err())
+		}
+		isUploaded := attrs.Status != b2.Uploaded
+		fileChan <- File{Path: path.Base(fileName), Size: int(attrs.Size), IsUploaded: &isUploaded, ModTime: attrs.UploadTimestamp}
 	}
 
 	return nil
 }
 
 var ErrLocalNotInCloud error = errors.New("exists locally but not in the cloud")
-var ErrCloudNotInLocal error = errors.New("exists on cloud but not locally")
+var ErrCloudNotInLocal error = errors.New("exists on B2 but not locally")
 
 type B2Local struct {
-	File       string
+	File File
 	Err  error
-	LocalCount int
 }
 
 // CompareConcurrent concurrently fetches the list of local files and cloud files,
 // then compares them to ensure all local files exist in the cloud.
 // Errors are sent to a provided error channel. The function will panic if an error occurs while listing files.
-func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName string, msgChan chan<- B2Local) {
+func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName string, localChan chan<- B2Local, b2Chan chan<- B2Local, doneChan chan<- interface{}) {
 	var wg sync.WaitGroup
-	localFiles := make(map[string]int)
+	localFiles := make(map[string]File)
-	cloudFiles := make(map[string]int)
+	cloudFiles := make(map[string]File)
-	localFileChan := make(chan string)
+	localFileChan := make(chan File)
-	b2FileChan := make(chan string)
+	b2FileChan := make(chan File)
 
 	// Local listing
 	wg.Add(1)
@@ -93,11 +108,11 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName
 		go func() {
 			defer wg.Done()
 			for f := range localFileChan {
-				if _, ok := localFiles[f]; ok {
+				if _, ok := localFiles[f.Path]; ok {
-					panic(fmt.Errorf("local file already exists in map: %s", f))
+					panic(fmt.Errorf("local file already exists in map: %s", f.Path))
 				}
-				b.logger.Debugln("local file ", f)
+				b.logger.Debugf("local file %+v\n", f)
-				localFiles[f]++
+				localFiles[f.Path] = f
 			}
 		}()
 
@@ -114,11 +129,11 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName
 		go func() {
 			defer wg.Done()
 			for f := range b2FileChan {
-				if _, ok := cloudFiles[f]; ok {
+				if _, ok := cloudFiles[f.Path]; ok {
-					panic(fmt.Errorf("cloud file already exists in map: %s", f))
+					panic(fmt.Errorf(`cloud file already exists in map: %s\n you should run 'backblazebackup cleanup --bucket "%s"'`, f.Path, b.bucketName))
 				}
-				b.logger.Debugln("B2 file ", f)
+				b.logger.Debugf("B2 file %+v\n", f)
-				cloudFiles[f]++
+				cloudFiles[f.Path] = f
 			}
 		}()
 		if err := b.b2BucketFiles(ctx, bucketName, b2FileChan); err != nil {
@@ -133,27 +148,29 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName
 	wg.Add(2)
 	go func() {
 		defer wg.Done()
-		for localFile := range localFiles {
+		for path, localFile := range localFiles {
-			if _, exists := cloudFiles[localFile]; !exists {
+			if _, exists := cloudFiles[path]; !exists {
-				msgChan <- B2Local{File: localFile, Err: ErrLocalNotInCloud}
+				localChan <- B2Local{File: localFile, Err: ErrLocalNotInCloud}
 				continue
 			}
-			msgChan <- B2Local{File: localFile, Err: nil}
+			b.logger.Debugf("localFile %+v\n", localFile)
+			localChan <- B2Local{File: localFile, Err: nil}
 		}
 	}()
 
 	// Now check cloud files that are not in local
 	go func() {
 		defer wg.Done()
-		for cloudFile := range cloudFiles {
+		for path, cloudFile := range cloudFiles {
-			b.logger.Debugln("cloudFile ", cloudFile)
+			if _, exists := localFiles[path]; !exists {
-			if _, exists := localFiles[cloudFile]; !exists {
+				b2Chan <- B2Local{File: cloudFile, Err: ErrCloudNotInLocal}
-				msgChan <- B2Local{File: cloudFile, Err: ErrCloudNotInLocal}
+				continue
 			}
+			b2Chan <- B2Local{File: cloudFile, Err: nil}
 		}
 	}()
-
 	wg.Wait()
-	msgChan <- B2Local{Err: nil, LocalCount: len(localFiles)}
+	close(localChan)
-	close(msgChan)
+	close(b2Chan)
+	doneChan <- nil
 }

internal/services/email/email.go

@@ -32,10 +32,7 @@ type EmailWithAttachments struct {
 	To          string
 	Bucket      string
 	BackupDir   string
-	CountNotInLocal int
+	Count       map[string]int
-	CountOK         int
-	CountNotInCloud int
-	CountLocal      int
 	Attachments []EmailAttachment
 }
 
@@ -75,11 +72,9 @@ func NewMailService(config MailServiceConfig) *EmailService {
 func (e *EmailService) SendOK(emailData EmailWithAttachments) error {
 	template := strings.Replace(htmlTemplate, "{{bucket}}", emailData.Bucket, -1)
 	template = strings.Replace(template, "{{local_backup_path}}", emailData.BackupDir, -1)
-	template = strings.Replace(template, "{{count_ErrCloudNotInLocal}}", fmt.Sprint(emailData.CountNotInLocal), -1)
+	for k, v := range emailData.Count {
-	template = strings.Replace(template, "{{count_ErrLocalNotInCloud}}", fmt.Sprint(emailData.CountNotInCloud), -1)
+		template = strings.Replace(template, "{{"+k+"}}", fmt.Sprint(v), -1)
-	template = strings.Replace(template, "{{count_ok}}", fmt.Sprint(emailData.CountOK), -1)
+	}
-	template = strings.Replace(template, "{{count_local}}", fmt.Sprint(emailData.CountLocal), -1)
-
 	msg, err := newMessage(e.from, emailData.To, subjectReport).
 		withAttachments(template, emailData.Attachments)
 

internal/services/email/template.go

@@ -11,11 +11,12 @@ const htmlTemplate = `<!DOCTYPE html>
             <h2 style="color: #333333; margin-top: 0;">Bucket Report</h2>
             <hr style="border: none; border-bottom: 1px solid #ddd;">
             <p><strong>Local Backup Path:</strong> {{local_backup_path}}</p>
-            <p><strong>Local files:</strong> {{count_local}}</p>
             <p><strong>Bucket Name:</strong> {{bucket}}</p>
+            <p><strong>Local files:</strong> {{count_local}}</p>
+            <p><strong>B2 files:</strong> {{count_b2}}</p>
             <p><strong>OK local in B2:</strong> {{count_ok}}</p>
-            <p><strong>B2 files not in local:</strong> {{count_ErrCloudNotInLocal}}</p>
+            <p><strong>B2 files not in local:</strong> {{count_not_in_local}}</p>
-            <p><strong>Local files not in B2:</strong> {{count_ErrLocalNotInCloud}}</p>
+            <p><strong>Local files not in B2:</strong> {{count_not_in_cloud}}</p>
             <hr style="border: none; border-bottom: 1px solid #ddd;">
             <p style="text-align: center; color: #666666;">This is an automated report, please do not reply to this email.</p>
         </div>