feat: improve report add columns to display modtime and size
This commit is contained in:
parent
ecc3d7fa64
commit
bec9807a75
81
cmd/check.go
81
cmd/check.go
|
|
@ -3,9 +3,9 @@ package cmd
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"net/smtp"
|
||||
"os"
|
||||
"strings"
|
||||
|
|
@ -72,20 +72,27 @@ var Check = &cobra.Command{
|
|||
|
||||
logger.Info("start check")
|
||||
defer logger.Info("end check")
|
||||
msgChan := make(chan backblaze.B2Local)
|
||||
go b.CompareConcurrent(ctx, backupDir, bucketName, msgChan)
|
||||
localChan := make(chan backblaze.B2Local)
|
||||
b2Chan := make(chan backblaze.B2Local)
|
||||
doneChan := make(chan interface{}, 1)
|
||||
go b.CompareConcurrent(ctx, backupDir, bucketName, localChan, b2Chan, doneChan)
|
||||
|
||||
reportBuilder := strings.Builder{}
|
||||
countLocal := 0
|
||||
countB2 := 0
|
||||
countOK := 0
|
||||
reportBuilder.WriteString(fmt.Sprintf("Local files within `%s` path already in `%s` bucket:\n", backupDir, bucketName))
|
||||
|
||||
cloudBuilder := strings.Builder{}
|
||||
cloudBuilder.WriteString(fmt.Sprintf("List of B2 files within `%s` bucket not found in local path `%s`\n", bucketName, backupDir))
|
||||
cloudBuilder.WriteString(fmt.Sprintf("B2 files within `%s` bucket\n", bucketName))
|
||||
cloudBuilder.WriteString("| B2 File Name | Size | ModTime | Status |\n")
|
||||
cloudBuilder.WriteString("|----------------------------------------------------------|------------------|--------------------------------|------------------------------------------\n")
|
||||
countNotInLocal := 0
|
||||
|
||||
localBuilder := strings.Builder{}
|
||||
localBuilder.WriteString(fmt.Sprintf("List of local files in `%s` not found in B2 bucket `%s`\n", backupDir, bucketName))
|
||||
localBuilder.WriteString(fmt.Sprintf("Local files in `%s`\n", backupDir))
|
||||
localBuilder.WriteString("| Local File Name | Size | ModTime | Status |\n")
|
||||
localBuilder.WriteString("|----------------------------------------------------------|------------------|--------------------------------|------------------------------------------\n")
|
||||
countNotInCloud := 0
|
||||
|
||||
loop:
|
||||
|
|
@ -93,55 +100,73 @@ var Check = &cobra.Command{
|
|||
select {
|
||||
case <-ctx.Done():
|
||||
// Handle the timeout or cancellation
|
||||
// Release any resources here
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
logger.Error("Operation timed out")
|
||||
} else if ctx.Err() == context.Canceled {
|
||||
logger.Error("Operation canceled")
|
||||
}
|
||||
break loop
|
||||
case msg := <-msgChan:
|
||||
if msg.Err == nil && msg.File == "" {
|
||||
countLocal = msg.LocalCount
|
||||
case <-doneChan:
|
||||
logger.Debugln("done chan")
|
||||
break loop
|
||||
}
|
||||
if errors.Is(msg.Err, backblaze.ErrCloudNotInLocal) {
|
||||
logger.Debug(msg.File + ": B2 file not found in local")
|
||||
cloudBuilder.WriteString(msg.File + "\n")
|
||||
countNotInLocal++
|
||||
case localMsg, ok := <-localChan:
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if errors.Is(msg.Err, backblaze.ErrLocalNotInCloud) {
|
||||
logger.Debug(msg.File + ": local file not found in B2")
|
||||
localBuilder.WriteString(msg.File + "\n")
|
||||
countNotInCloud++
|
||||
countLocal++
|
||||
logger.Debugln("localChan", localMsg.File)
|
||||
writeTableRow(&localBuilder, &countNotInCloud, localMsg.File, localMsg.Err)
|
||||
case b2Msg, ok := <-b2Chan:
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
reportBuilder.WriteString(msg.File + " OK" + "\n")
|
||||
countOK++
|
||||
countB2++
|
||||
logger.Debugln("cloudBuilder", b2Msg.File)
|
||||
writeTableRow(&cloudBuilder, &countNotInLocal, b2Msg.File, b2Msg.Err)
|
||||
}
|
||||
}
|
||||
|
||||
if countNotInLocal > 0 {
|
||||
reportBuilder.WriteString("\n")
|
||||
reportBuilder.WriteString(cloudBuilder.String())
|
||||
}
|
||||
if countNotInCloud > 0 {
|
||||
reportBuilder.WriteString("\n")
|
||||
reportBuilder.WriteString(localBuilder.String())
|
||||
}
|
||||
|
||||
if err := mailSrv.SendOK(email.EmailWithAttachments{
|
||||
To: cfg.MailTo,
|
||||
Bucket: bucketName,
|
||||
BackupDir: backupDir,
|
||||
CountNotInLocal: countNotInLocal,
|
||||
CountNotInCloud: countNotInCloud,
|
||||
CountOK: countOK,
|
||||
CountLocal: countLocal,
|
||||
Count: map[string]int{
|
||||
"count_ok": countOK,
|
||||
"count_local": countLocal,
|
||||
"count_b2": countB2,
|
||||
"count_not_in_local": countNotInLocal,
|
||||
"count_not_in_cloud": countNotInCloud,
|
||||
},
|
||||
Attachments: []email.EmailAttachment{{File: bytes.NewReader([]byte(reportBuilder.String())), Title: fmt.Sprintf("%s-check-report.txt", bucketName)}},
|
||||
}); err != nil {
|
||||
panic(fmt.Errorf("error while send email: %w", err))
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
// New function to write a table row
|
||||
func writeTableRow(builder *strings.Builder, count *int, file backblaze.File, err error) {
|
||||
status := "OK"
|
||||
if err != nil {
|
||||
status = err.Error()
|
||||
(*count)++
|
||||
}
|
||||
|
||||
builder.WriteString(fmt.Sprintf("| %-56s | %-16s | %-30s | %-40s |\n", file.Path, prettyByteSize(file.Size), file.ModTime.Format("2006-01-02 15:04:05.000"), status))
|
||||
}
|
||||
|
||||
func prettyByteSize(b int) string {
|
||||
bf := float64(b)
|
||||
for _, unit := range []string{"", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"} {
|
||||
if math.Abs(bf) < 1024.0 {
|
||||
return fmt.Sprintf("%3.1f%sB", bf, unit)
|
||||
}
|
||||
bf /= 1024.0
|
||||
}
|
||||
return fmt.Sprintf("%.1fYiB", bf)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_prettyByteSize(t *testing.T) {
|
||||
require.Equal(t, prettyByteSize(1505099776), "1.4GiB")
|
||||
}
|
||||
|
|
@ -8,13 +8,21 @@ import (
|
|||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/kurin/blazer/b2"
|
||||
)
|
||||
|
||||
type File struct {
|
||||
Path string
|
||||
Size int // file size in bytes
|
||||
ModTime time.Time
|
||||
IsUploaded *bool
|
||||
}
|
||||
|
||||
// localFiles lists the local files in the given backup directory and sends them to a channel.
|
||||
// It closes the channel after all files are listed.
|
||||
func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- string) error {
|
||||
func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- File) error {
|
||||
defer close(fileChan)
|
||||
// Walk the directory and send files to the channel
|
||||
err := filepath.WalkDir(backupDir, func(path string, d fs.DirEntry, err error) error {
|
||||
|
|
@ -22,12 +30,16 @@ func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- string) error {
|
|||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
fileChan <- filepath.Base(path)
|
||||
i, err := d.Info()
|
||||
if err != nil {
|
||||
return fmt.Errorf("d.Info: %w", err)
|
||||
}
|
||||
fileChan <- File{Path: filepath.Base(path), Size: int(i.Size()), ModTime: i.ModTime()}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking the directory: %v", err)
|
||||
return fmt.Errorf("error walking the directory: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -35,7 +47,7 @@ func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- string) error {
|
|||
|
||||
// b2BucketFiles lists the files in the given B2 bucket and sends them to a channel.
|
||||
// It closes the channel after all files are listed.
|
||||
func (b *BackBlaze) b2BucketFiles(ctx context.Context, bucketName string, fileChan chan<- string) error {
|
||||
func (b *BackBlaze) b2BucketFiles(ctx context.Context, bucketName string, fileChan chan<- File) error {
|
||||
bucket, err := b.b2Client.Bucket(ctx, bucketName)
|
||||
defer close(fileChan)
|
||||
if err != nil {
|
||||
|
|
@ -58,32 +70,35 @@ func (b *BackBlaze) b2BucketFiles(ctx context.Context, bucketName string, fileCh
|
|||
return errors.New("bucketIter Object is nil")
|
||||
}
|
||||
|
||||
// Retrieve just filename
|
||||
b.logger.Debugln("bucket file: ", path.Base(bucketIter.Object().Name()))
|
||||
fileChan <- path.Base(bucketIter.Object().Name())
|
||||
fileName := path.Base(bucketIter.Object().Name())
|
||||
attrs, err := bucketIter.Object().Attrs(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bucketIter.Object().Attrs %s err %w", fileName, bucketIter.Err())
|
||||
}
|
||||
isUploaded := attrs.Status != b2.Uploaded
|
||||
fileChan <- File{Path: path.Base(fileName), Size: int(attrs.Size), IsUploaded: &isUploaded, ModTime: attrs.UploadTimestamp}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var ErrLocalNotInCloud error = errors.New("exists locally but not in the cloud")
|
||||
var ErrCloudNotInLocal error = errors.New("exists on cloud but not locally")
|
||||
var ErrCloudNotInLocal error = errors.New("exists on B2 but not locally")
|
||||
|
||||
type B2Local struct {
|
||||
File string
|
||||
File File
|
||||
Err error
|
||||
LocalCount int
|
||||
}
|
||||
|
||||
// CompareConcurrent concurrently fetches the list of local files and cloud files,
|
||||
// then compares them to ensure all local files exist in the cloud.
|
||||
// Errors are sent to a provided error channel. The function will panic if an error occurs while listing files.
|
||||
func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName string, msgChan chan<- B2Local) {
|
||||
func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName string, localChan chan<- B2Local, b2Chan chan<- B2Local, doneChan chan<- interface{}) {
|
||||
var wg sync.WaitGroup
|
||||
localFiles := make(map[string]int)
|
||||
cloudFiles := make(map[string]int)
|
||||
localFileChan := make(chan string)
|
||||
b2FileChan := make(chan string)
|
||||
localFiles := make(map[string]File)
|
||||
cloudFiles := make(map[string]File)
|
||||
localFileChan := make(chan File)
|
||||
b2FileChan := make(chan File)
|
||||
|
||||
// Local listing
|
||||
wg.Add(1)
|
||||
|
|
@ -93,11 +108,11 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName
|
|||
go func() {
|
||||
defer wg.Done()
|
||||
for f := range localFileChan {
|
||||
if _, ok := localFiles[f]; ok {
|
||||
panic(fmt.Errorf("local file already exists in map: %s", f))
|
||||
if _, ok := localFiles[f.Path]; ok {
|
||||
panic(fmt.Errorf("local file already exists in map: %s", f.Path))
|
||||
}
|
||||
b.logger.Debugln("local file ", f)
|
||||
localFiles[f]++
|
||||
b.logger.Debugf("local file %+v\n", f)
|
||||
localFiles[f.Path] = f
|
||||
}
|
||||
}()
|
||||
|
||||
|
|
@ -114,11 +129,11 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName
|
|||
go func() {
|
||||
defer wg.Done()
|
||||
for f := range b2FileChan {
|
||||
if _, ok := cloudFiles[f]; ok {
|
||||
panic(fmt.Errorf("cloud file already exists in map: %s", f))
|
||||
if _, ok := cloudFiles[f.Path]; ok {
|
||||
panic(fmt.Errorf(`cloud file already exists in map: %s\n you should run 'backblazebackup cleanup --bucket "%s"'`, f.Path, b.bucketName))
|
||||
}
|
||||
b.logger.Debugln("B2 file ", f)
|
||||
cloudFiles[f]++
|
||||
b.logger.Debugf("B2 file %+v\n", f)
|
||||
cloudFiles[f.Path] = f
|
||||
}
|
||||
}()
|
||||
if err := b.b2BucketFiles(ctx, bucketName, b2FileChan); err != nil {
|
||||
|
|
@ -133,27 +148,29 @@ func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName
|
|||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for localFile := range localFiles {
|
||||
if _, exists := cloudFiles[localFile]; !exists {
|
||||
msgChan <- B2Local{File: localFile, Err: ErrLocalNotInCloud}
|
||||
for path, localFile := range localFiles {
|
||||
if _, exists := cloudFiles[path]; !exists {
|
||||
localChan <- B2Local{File: localFile, Err: ErrLocalNotInCloud}
|
||||
continue
|
||||
}
|
||||
msgChan <- B2Local{File: localFile, Err: nil}
|
||||
b.logger.Debugf("localFile %+v\n", localFile)
|
||||
localChan <- B2Local{File: localFile, Err: nil}
|
||||
}
|
||||
}()
|
||||
|
||||
// Now check cloud files that are not in local
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for cloudFile := range cloudFiles {
|
||||
b.logger.Debugln("cloudFile ", cloudFile)
|
||||
if _, exists := localFiles[cloudFile]; !exists {
|
||||
msgChan <- B2Local{File: cloudFile, Err: ErrCloudNotInLocal}
|
||||
for path, cloudFile := range cloudFiles {
|
||||
if _, exists := localFiles[path]; !exists {
|
||||
b2Chan <- B2Local{File: cloudFile, Err: ErrCloudNotInLocal}
|
||||
continue
|
||||
}
|
||||
b2Chan <- B2Local{File: cloudFile, Err: nil}
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
msgChan <- B2Local{Err: nil, LocalCount: len(localFiles)}
|
||||
close(msgChan)
|
||||
close(localChan)
|
||||
close(b2Chan)
|
||||
doneChan <- nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,10 +32,7 @@ type EmailWithAttachments struct {
|
|||
To string
|
||||
Bucket string
|
||||
BackupDir string
|
||||
CountNotInLocal int
|
||||
CountOK int
|
||||
CountNotInCloud int
|
||||
CountLocal int
|
||||
Count map[string]int
|
||||
Attachments []EmailAttachment
|
||||
}
|
||||
|
||||
|
|
@ -75,11 +72,9 @@ func NewMailService(config MailServiceConfig) *EmailService {
|
|||
func (e *EmailService) SendOK(emailData EmailWithAttachments) error {
|
||||
template := strings.Replace(htmlTemplate, "{{bucket}}", emailData.Bucket, -1)
|
||||
template = strings.Replace(template, "{{local_backup_path}}", emailData.BackupDir, -1)
|
||||
template = strings.Replace(template, "{{count_ErrCloudNotInLocal}}", fmt.Sprint(emailData.CountNotInLocal), -1)
|
||||
template = strings.Replace(template, "{{count_ErrLocalNotInCloud}}", fmt.Sprint(emailData.CountNotInCloud), -1)
|
||||
template = strings.Replace(template, "{{count_ok}}", fmt.Sprint(emailData.CountOK), -1)
|
||||
template = strings.Replace(template, "{{count_local}}", fmt.Sprint(emailData.CountLocal), -1)
|
||||
|
||||
for k, v := range emailData.Count {
|
||||
template = strings.Replace(template, "{{"+k+"}}", fmt.Sprint(v), -1)
|
||||
}
|
||||
msg, err := newMessage(e.from, emailData.To, subjectReport).
|
||||
withAttachments(template, emailData.Attachments)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,11 +11,12 @@ const htmlTemplate = `<!DOCTYPE html>
|
|||
<h2 style="color: #333333; margin-top: 0;">Bucket Report</h2>
|
||||
<hr style="border: none; border-bottom: 1px solid #ddd;">
|
||||
<p><strong>Local Backup Path:</strong> {{local_backup_path}}</p>
|
||||
<p><strong>Local files:</strong> {{count_local}}</p>
|
||||
<p><strong>Bucket Name:</strong> {{bucket}}</p>
|
||||
<p><strong>Local files:</strong> {{count_local}}</p>
|
||||
<p><strong>B2 files:</strong> {{count_b2}}</p>
|
||||
<p><strong>OK local in B2:</strong> {{count_ok}}</p>
|
||||
<p><strong>B2 files not in local:</strong> {{count_ErrCloudNotInLocal}}</p>
|
||||
<p><strong>Local files not in B2:</strong> {{count_ErrLocalNotInCloud}}</p>
|
||||
<p><strong>B2 files not in local:</strong> {{count_not_in_local}}</p>
|
||||
<p><strong>Local files not in B2:</strong> {{count_not_in_cloud}}</p>
|
||||
<hr style="border: none; border-bottom: 1px solid #ddd;">
|
||||
<p style="text-align: center; color: #666666;">This is an automated report, please do not reply to this email.</p>
|
||||
</div>
|
||||
|
|
|
|||
Loading…
Reference in New Issue