parent
5bbeb8c392
commit
61984470da
|
|
@ -1,4 +1,5 @@
|
|||
.env
|
||||
.env.*
|
||||
.vscode
|
||||
coverage
|
||||
.notes
|
||||
|
|
|
|||
|
|
@ -0,0 +1,134 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/smtp"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"gitea.urkob.com/urko/backblaze-backup/internal/services/backblaze"
|
||||
"gitea.urkob.com/urko/backblaze-backup/internal/services/email"
|
||||
"gitea.urkob.com/urko/backblaze-backup/kit"
|
||||
"gitea.urkob.com/urko/backblaze-backup/kit/config"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var Check = &cobra.Command{
|
||||
Use: "check",
|
||||
Short: "Compares local backup files with those in a Backblaze B2 bucket and sends a summary email.",
|
||||
Long: `This command compares the list of files in a local backup directory against the files in a specified Backblaze B2 bucket.
|
||||
The operation is performed concurrently and is time-bound, set by default to 5 minutes.
|
||||
If discrepancies are found, i.e., some files exist only locally or only on the cloud, these are logged and sent via email as attachments.
|
||||
|
||||
The email contains two text attachments:
|
||||
- 'Local-Files-Not-In-B2.txt': Lists files that are in the local backup directory but not in the B2 bucket.
|
||||
- 'B2-Files-Not-In-Local.txt': Lists files that are in the B2 bucket but not in the local backup directory.
|
||||
|
||||
The command requires two flags:
|
||||
- '--dir': The path of the local backup directory
|
||||
- '--bucket': The name of the Backblaze B2 bucket
|
||||
|
||||
An environment variable 'BACKBLAZE_ENV' can be set to 'dev' to load environment variables from a .env file in the root directory.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) // or any appropriate time
|
||||
defer cancel()
|
||||
|
||||
log.SetFlags(log.Ldate | log.Lmicroseconds)
|
||||
backupDir, err := cmd.Flags().GetString("dir")
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("dir %w", err))
|
||||
}
|
||||
bucketName, err := cmd.Flags().GetString("bucket")
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("bucket %w", err))
|
||||
}
|
||||
|
||||
envFile := ""
|
||||
if os.Getenv("BACKBLAZE_ENV") == "dev" {
|
||||
envFile = kit.RootDir() + "/.env"
|
||||
}
|
||||
|
||||
cfg := config.NewConfig(envFile)
|
||||
mailSrv := email.NewMailService(email.MailServiceConfig{
|
||||
Auth: smtp.PlainAuth("", cfg.MailUser, cfg.MailPassword, cfg.MailHost),
|
||||
Host: cfg.MailHost,
|
||||
Port: cfg.MailPort,
|
||||
From: cfg.MailUser,
|
||||
},
|
||||
)
|
||||
logger := logrus.New()
|
||||
logger.SetLevel(logrus.Level(cfg.LogLevel))
|
||||
|
||||
b, err := backblaze.NewBackBlaze(ctx, logger, cfg.BbId, cfg.BbKey)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("NewBackBlaze %w", err))
|
||||
}
|
||||
|
||||
logger.Info("start check")
|
||||
defer logger.Info("end check")
|
||||
errs := make(chan backblaze.B2LocalErr)
|
||||
go b.CompareConcurrent(ctx, backupDir, bucketName, errs)
|
||||
|
||||
// Create two buffers for two kinds of errors
|
||||
cloudNotInLocalBuffer := new(bytes.Buffer)
|
||||
cloudNotInLocalBuffer.WriteString(fmt.Sprintf("List of B2 files within %s bucket not found in local path %s \n", bucketName, backupDir))
|
||||
countNotInLocalBuffer := 0
|
||||
|
||||
localNotInCloudBuffer := new(bytes.Buffer)
|
||||
localNotInCloudBuffer.WriteString(fmt.Sprintf("List of local files in %s not found in B2 bucket %s \n", backupDir, bucketName))
|
||||
countNotInCloudBuffer := 0
|
||||
|
||||
loop:
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Handle the timeout or cancellation
|
||||
// Release any resources here
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
logger.Error("Operation timed out")
|
||||
} else if ctx.Err() == context.Canceled {
|
||||
logger.Error("Operation canceled")
|
||||
}
|
||||
break loop
|
||||
case err := <-errs:
|
||||
if errors.Is(err.Err, backblaze.ErrCloudNotInLocal) {
|
||||
logger.Debug(err.File + ": B2 file not found in local")
|
||||
cloudNotInLocalBuffer.WriteString(err.File + "\n")
|
||||
countNotInLocalBuffer++
|
||||
}
|
||||
if errors.Is(err.Err, backblaze.ErrLocalNotInCloud) {
|
||||
logger.Debug(err.File + ": local file not found in B2")
|
||||
localNotInCloudBuffer.WriteString(err.File + "\n")
|
||||
countNotInCloudBuffer++
|
||||
}
|
||||
if err.Err == nil {
|
||||
break loop
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var attachments []email.EmailAttachment
|
||||
if countNotInCloudBuffer > 0 {
|
||||
attachments = append(attachments, email.EmailAttachment{File: cloudNotInLocalBuffer, Title: "B2-Files-Not-In-Local.txt"})
|
||||
}
|
||||
if countNotInLocalBuffer > 0 {
|
||||
attachments = append(attachments, email.EmailAttachment{File: localNotInCloudBuffer, Title: "Local-Files-Not-In-B2.txt"})
|
||||
}
|
||||
|
||||
if err := mailSrv.SendOK(email.EmailWithAttachments{
|
||||
To: cfg.MailTo,
|
||||
Bucket: bucketName,
|
||||
BackupDir: backupDir,
|
||||
CountLocal: countNotInLocalBuffer,
|
||||
CountCloud: countNotInCloudBuffer,
|
||||
Attachments: attachments,
|
||||
}); err != nil {
|
||||
panic(fmt.Errorf("error while send email: %w", err))
|
||||
}
|
||||
},
|
||||
}
|
||||
|
|
@ -2,11 +2,14 @@ package cmd
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"gitea.urkob.com/urko/backblaze-backup/internal/services"
|
||||
"gitea.urkob.com/urko/backblaze-backup/internal/services/backblaze"
|
||||
"gitea.urkob.com/urko/backblaze-backup/kit"
|
||||
"gitea.urkob.com/urko/backblaze-backup/kit/config"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
@ -20,18 +23,23 @@ var Cleanup = &cobra.Command{
|
|||
|
||||
bucketName, err := cmd.Flags().GetString("bucket")
|
||||
if err != nil {
|
||||
log.Fatalln("bucket %w", err)
|
||||
panic(fmt.Errorf("bucket %w", err))
|
||||
}
|
||||
|
||||
envFile := ""
|
||||
if os.Getenv("BACKBLAZE_ENV") == "dev" {
|
||||
envFile = ".env"
|
||||
envFile = kit.RootDir() + "/.env"
|
||||
}
|
||||
cfg := config.NewConfig(envFile)
|
||||
logger := logrus.New()
|
||||
logger.SetLevel(logrus.Level(cfg.LogLevel))
|
||||
|
||||
bbService := services.NewBackBlaze(cfg.BbId, cfg.BbKey)
|
||||
bbService, err := backblaze.NewBackBlaze(ctx, logger, cfg.BbId, cfg.BbKey)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("NewBackBlaze %w", err))
|
||||
}
|
||||
if err := bbService.CleanUp(ctx, cancel, bucketName); err != nil {
|
||||
log.Fatalln("bbService.CleanUp()", err)
|
||||
panic(fmt.Errorf("bbService.CleanUp %w", err))
|
||||
}
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,13 +2,16 @@ package cmd
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"gitea.urkob.com/urko/backblaze-backup/internal/services"
|
||||
"gitea.urkob.com/urko/backblaze-backup/internal/services/backblaze"
|
||||
"gitea.urkob.com/urko/backblaze-backup/kit"
|
||||
"gitea.urkob.com/urko/backblaze-backup/kit/config"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
@ -22,13 +25,18 @@ var Versions = &cobra.Command{
|
|||
|
||||
envFile := ""
|
||||
if os.Getenv("BACKBLAZE_ENV") == "dev" {
|
||||
envFile = ".env"
|
||||
envFile = kit.RootDir() + "/.env"
|
||||
}
|
||||
cfg := config.NewConfig(envFile)
|
||||
logger := logrus.New()
|
||||
logger.SetLevel(logrus.Level(cfg.LogLevel))
|
||||
|
||||
bbService := services.NewBackBlaze(cfg.BbId, cfg.BbKey)
|
||||
bbService, err := backblaze.NewBackBlaze(ctx, logger, cfg.BbId, cfg.BbKey)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("NewBackBlaze %w", err))
|
||||
}
|
||||
if err := bbService.ListDuplicateVersions(ctx, cancel); err != nil {
|
||||
log.Fatalln("bbService.ListDuplicateVersions()", err)
|
||||
panic(fmt.Errorf("bbService.ListDuplicateVersions %w", err))
|
||||
}
|
||||
},
|
||||
}
|
||||
|
|
|
|||
27
cmd/sync.go
27
cmd/sync.go
|
|
@ -2,11 +2,14 @@ package cmd
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"gitea.urkob.com/urko/backblaze-backup/internal/services"
|
||||
"gitea.urkob.com/urko/backblaze-backup/internal/services/backblaze"
|
||||
"gitea.urkob.com/urko/backblaze-backup/kit"
|
||||
"gitea.urkob.com/urko/backblaze-backup/kit/config"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
@ -21,26 +24,34 @@ var Sync = &cobra.Command{
|
|||
log.SetFlags(log.Ldate | log.Lmicroseconds)
|
||||
filePath, err := cmd.Flags().GetString("file")
|
||||
if err != nil {
|
||||
log.Fatalln("file %w", err)
|
||||
panic(fmt.Errorf("file %w", err))
|
||||
}
|
||||
dir, err := cmd.Flags().GetString("dir")
|
||||
if err != nil {
|
||||
log.Fatalln("dir %w", err)
|
||||
panic(fmt.Errorf("dir %w", err))
|
||||
}
|
||||
bucketName, err := cmd.Flags().GetString("bucket")
|
||||
if err != nil {
|
||||
log.Fatalln("bucket %w", err)
|
||||
panic(fmt.Errorf("bucket %w", err))
|
||||
}
|
||||
|
||||
envFile := ""
|
||||
if os.Getenv("BACKBLAZE_ENV") == "dev" {
|
||||
envFile = ".env"
|
||||
envFile = kit.RootDir() + "/.env"
|
||||
}
|
||||
cfg := config.NewConfig(envFile)
|
||||
|
||||
bbService := services.NewBackBlaze(cfg.BbId, cfg.BbKey).WithBucket(bucketName).WithDir(dir).WithFile(filePath)
|
||||
cfg := config.NewConfig(envFile)
|
||||
logger := logrus.New()
|
||||
logger.SetLevel(logrus.Level(cfg.LogLevel))
|
||||
|
||||
bbService, err := backblaze.NewBackBlaze(ctx, logger, cfg.BbId, cfg.BbKey)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("NewBackBlaze %w", err))
|
||||
}
|
||||
|
||||
bbService = bbService.WithBucket(bucketName).WithDir(dir).WithFile(filePath)
|
||||
if err := bbService.Sync(ctx); err != nil {
|
||||
log.Fatalln("bbService.Sync()", err)
|
||||
panic(fmt.Errorf("bbService.Sync %w", err))
|
||||
}
|
||||
},
|
||||
}
|
||||
|
|
|
|||
16
go.mod
16
go.mod
|
|
@ -9,18 +9,22 @@ require (
|
|||
github.com/kurin/blazer v0.5.3
|
||||
github.com/rclone/rclone v1.63.1
|
||||
github.com/spf13/cobra v1.7.0
|
||||
github.com/stretchr/testify v1.8.3
|
||||
golang.org/x/sync v0.3.0
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/go-chi/chi/v5 v5.0.8 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.11.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
|
|
@ -28,6 +32,7 @@ require (
|
|||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/prometheus/client_golang v1.14.0 // indirect
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
|
|
@ -41,11 +46,14 @@ require (
|
|||
github.com/tklauser/go-sysconf v0.3.11 // indirect
|
||||
github.com/tklauser/numcpus v0.6.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.3 // indirect
|
||||
golang.org/x/crypto v0.7.0 // indirect
|
||||
golang.org/x/net v0.8.0 // indirect
|
||||
golang.org/x/crypto v0.9.0 // indirect
|
||||
golang.org/x/net v0.10.0 // indirect
|
||||
golang.org/x/oauth2 v0.8.0 // indirect
|
||||
golang.org/x/sys v0.8.0 // indirect
|
||||
golang.org/x/term v0.7.0 // indirect
|
||||
golang.org/x/text v0.8.0 // indirect
|
||||
golang.org/x/term v0.8.0 // indirect
|
||||
golang.org/x/text v0.9.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
google.golang.org/api v0.126.0 // indirect
|
||||
google.golang.org/protobuf v1.30.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
|
|||
38
go.sum
38
go.sum
|
|
@ -12,6 +12,7 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP
|
|||
cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
|
|
@ -19,8 +20,9 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
|
|||
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ=
|
||||
cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
|
||||
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
|
||||
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
|
|
@ -161,11 +163,13 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
|
|||
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc=
|
||||
github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
|
||||
github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
|
||||
|
|
@ -208,8 +212,10 @@ github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 h1:uF5FHZ/L5gv
|
|||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 h1:FHVoZMOVRA+6/y4yRlbiR3WvsrOcKBd/f64H7YiWR2U=
|
||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kurin/blazer v0.5.3 h1:SAgYv0TKU0kN/ETfO5ExjNAPyMt2FocO2s/UlCHfjAk=
|
||||
github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
|
||||
|
|
@ -339,8 +345,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
|
|||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
|
||||
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
|
|
@ -402,8 +408,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
|
|||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -411,7 +417,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr
|
|||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
|
||||
golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
|
||||
golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
|
@ -470,8 +477,8 @@ golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
|
|||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
@ -479,8 +486,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
|||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
@ -546,7 +553,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
|
|||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.115.0 h1:6FFkVvStt4YqXSx3azKyzj7fXerGnVlLJ/eud01nBDE=
|
||||
google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o=
|
||||
google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
|
|
@ -583,7 +591,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
|
|||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 h1:0BOZf6qNozI3pkN3fJLwNubheHJYHhMh91GRFOWWK08=
|
||||
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
|
|
@ -596,7 +605,7 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
|
|||
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag=
|
||||
google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
|
|
@ -615,6 +624,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
|
|||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
|
|
|||
|
|
@ -1,364 +0,0 @@
|
|||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
rcloneb2 "github.com/rclone/rclone/backend/b2"
|
||||
|
||||
"github.com/kurin/blazer/b2"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
const writers = 4
|
||||
const maxConcurrentWeight = 5
|
||||
const largeFileSize = 500 * 1024 * 1024 // 500 MB
|
||||
|
||||
type BackBalze struct {
|
||||
bucketName string
|
||||
dir string
|
||||
filePath string
|
||||
maxWorkers int
|
||||
bbID string
|
||||
bbKey string
|
||||
}
|
||||
|
||||
func NewBackBlaze(bbID, bbKey string) *BackBalze {
|
||||
log.Println("runtime.NumCPU()", runtime.NumCPU())
|
||||
return &BackBalze{
|
||||
bbID: bbID,
|
||||
bbKey: bbKey,
|
||||
maxWorkers: runtime.NumCPU(),
|
||||
}
|
||||
}
|
||||
func (b *BackBalze) WithBucket(bucketName string) *BackBalze {
|
||||
b.bucketName = bucketName
|
||||
return b
|
||||
}
|
||||
func (b *BackBalze) WithDir(dir string) *BackBalze {
|
||||
b.dir = dir
|
||||
return b
|
||||
}
|
||||
func (b *BackBalze) WithFile(filePath string) *BackBalze {
|
||||
b.filePath = filePath
|
||||
return b
|
||||
}
|
||||
func (b *BackBalze) Sync(ctx context.Context) error {
|
||||
if b.bucketName == "" && (b.filePath == "" || b.dir == "") {
|
||||
return fmt.Errorf("bucket name is %v | filePath is %v | dir is %v", b.bucketName, b.filePath, b.dir)
|
||||
}
|
||||
|
||||
if b.filePath != "" && b.dir != "" {
|
||||
return errors.New("you must select just 1 option, dir or file")
|
||||
}
|
||||
|
||||
b2Client, err := b2.NewClient(ctx, b.bbID, b.bbKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b2.NewClient %w", err)
|
||||
}
|
||||
log.Println("b2Client ok")
|
||||
bc, err := b2Client.Bucket(ctx, b.bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b2Client.Bucket %w", err)
|
||||
}
|
||||
|
||||
if bc == nil {
|
||||
return fmt.Errorf("bucket doesn't exist %s", b.bucketName)
|
||||
}
|
||||
|
||||
log.Println("bucket found:", bc.Name())
|
||||
if b.filePath != "" {
|
||||
log.Println("file:", b.filePath)
|
||||
|
||||
if err := copyFile(ctx, bc, b.filePath); err != nil {
|
||||
return fmt.Errorf("copyFile %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if b.dir != "" {
|
||||
oldFiles, err := bucketFiles(ctx, bc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bucketFiles %w", err)
|
||||
}
|
||||
log.Println(strings.Repeat("*", 40))
|
||||
log.Println("oldFiles to clean:\n\t\t" + strings.Join(oldFiles, "\n\t\t"))
|
||||
log.Println(strings.Repeat("*", 40))
|
||||
|
||||
fileChan := make(chan string)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 0; i < b.maxWorkers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for src := range fileChan {
|
||||
err := copyFile(ctx, bc, src)
|
||||
if err != nil {
|
||||
log.Printf("error copying file %s: %v\n", src, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Walk the directory and send files to the channel for uploading
|
||||
err = filepath.WalkDir(b.dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
fileChan <- path
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking the directory: %v", err)
|
||||
}
|
||||
|
||||
// Close the channel (no more files to send)
|
||||
close(fileChan)
|
||||
wg.Wait()
|
||||
|
||||
// Cleanup old files after backup is completed
|
||||
if err := cleanBucket(ctx, bc, oldFiles); err != nil {
|
||||
return fmt.Errorf("cleanBucket %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("copied successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BackBalze) OldSync() error {
|
||||
if b.bucketName == "" && (b.filePath == "" || b.dir == "") {
|
||||
return fmt.Errorf("bucket name is %v | filePath is %v | dir is %v", b.bucketName, b.filePath, b.dir)
|
||||
}
|
||||
|
||||
if b.filePath != "" && b.dir != "" {
|
||||
return errors.New("you must select just 1 option, dir or file")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
b2Client, err := b2.NewClient(ctx, b.bbID, b.bbKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b2.NewClient %w", err)
|
||||
}
|
||||
log.Println("b2Client ok")
|
||||
bc, err := b2Client.Bucket(ctx, b.bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b2Client.Bucket %w", err)
|
||||
}
|
||||
|
||||
if bc == nil {
|
||||
return fmt.Errorf("bucket doesn't exist %s", b.bucketName)
|
||||
}
|
||||
|
||||
log.Println("bucket found:", bc.Name())
|
||||
if b.filePath != "" {
|
||||
log.Println("file:", b.filePath)
|
||||
|
||||
if err := copyFile(ctx, bc, b.filePath); err != nil {
|
||||
return fmt.Errorf("copyFile %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if b.dir != "" {
|
||||
oldFiles, err := bucketFiles(ctx, bc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bucketFiles %w", err)
|
||||
}
|
||||
log.Println(strings.Repeat("*", 40))
|
||||
log.Println("oldFiles to clean:\n\t\t" + strings.Join(oldFiles, "\n\t\t"))
|
||||
log.Println(strings.Repeat("*", 40))
|
||||
|
||||
fileChan := make(chan string)
|
||||
uploadSem := semaphore.NewWeighted(maxConcurrentWeight)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 0; i < b.maxWorkers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for src := range fileChan {
|
||||
info, err := os.Stat(src)
|
||||
if err != nil {
|
||||
log.Printf("error getting file info %s: %v\n", src, err)
|
||||
continue
|
||||
}
|
||||
weight := int64(1)
|
||||
if info.Size() > largeFileSize {
|
||||
weight = 2
|
||||
}
|
||||
if err := uploadSem.Acquire(ctx, weight); err == nil {
|
||||
log.Println("start copying file", src)
|
||||
if err := copyFile(ctx, bc, src); err != nil {
|
||||
log.Printf("error copying file %s: %v\n", src, err)
|
||||
}
|
||||
uploadSem.Release(weight)
|
||||
} else {
|
||||
log.Printf("error acquiring semaphore: %v\n", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Walk the directory and send files to the channel for uploading
|
||||
err = filepath.WalkDir(b.dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
fileChan <- path
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking the directory: %v", err)
|
||||
}
|
||||
|
||||
// Close the channel (no more files to send)
|
||||
close(fileChan)
|
||||
wg.Wait()
|
||||
|
||||
// Cleanup old files after backup is completed
|
||||
if err := cleanBucket(ctx, bc, oldFiles); err != nil {
|
||||
return fmt.Errorf("cleanBucket %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("copied successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyFile(ctx context.Context, bucket *b2.Bucket, src string) error {
|
||||
f, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w := bucket.Object(fi.Name()).NewWriter(ctx)
|
||||
w.ConcurrentUploads = writers
|
||||
w.UseFileBuffer = true
|
||||
log.Println("start copying", fi.Name())
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
w.Close()
|
||||
return err
|
||||
}
|
||||
log.Println("end copying", fi.Name())
|
||||
return w.Close()
|
||||
}
|
||||
|
||||
func cleanBucket(ctx context.Context, bucket *b2.Bucket, files []string) error {
|
||||
var errorBuilder strings.Builder
|
||||
for _, v := range files {
|
||||
obj := bucket.Object(v)
|
||||
if obj == nil {
|
||||
log.Println("object is nil", v)
|
||||
continue
|
||||
}
|
||||
if err := obj.Delete(ctx); err != nil {
|
||||
errorBuilder.WriteString(fmt.Errorf("error deleting %s : %w", v, err).Error())
|
||||
errorBuilder.WriteString("; ")
|
||||
}
|
||||
}
|
||||
|
||||
if errorBuilder.Len() > 0 {
|
||||
return errors.New(errorBuilder.String())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func bucketFiles(ctx context.Context, bucket *b2.Bucket) ([]string, error) {
|
||||
bucketIter := bucket.List(ctx)
|
||||
if bucketIter == nil {
|
||||
return nil, fmt.Errorf("bucket list cannot be nil")
|
||||
}
|
||||
var files []string
|
||||
for {
|
||||
if !bucketIter.Next() {
|
||||
if bucketIter.Err() != nil {
|
||||
return nil, fmt.Errorf("bucketIter err %w", bucketIter.Err())
|
||||
}
|
||||
break
|
||||
}
|
||||
if bucketIter.Object() == nil {
|
||||
log.Println("bucketIter Object is nil")
|
||||
continue
|
||||
}
|
||||
files = append(files, bucketIter.Object().Name())
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
type duplicate struct {
|
||||
bucket string
|
||||
file string
|
||||
count int
|
||||
}
|
||||
|
||||
func (b *BackBalze) CleanUp(ctx context.Context, cancel context.CancelFunc, bucketName string) error {
|
||||
|
||||
b2Client, err := b2.NewClient(ctx, b.bbID, b.bbKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b2.NewClient %w", err)
|
||||
}
|
||||
|
||||
log.Println("b2Client ok")
|
||||
|
||||
var dups []duplicate
|
||||
if bucketName != "" {
|
||||
dups, err = b.listDuplicatesFromBucket(ctx, cancel, b2Client, bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b.listDuplicatesFromBucket: %w", err)
|
||||
}
|
||||
} else {
|
||||
dups, err = b.listDuplicates(ctx, cancel, b2Client)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b.listDuplicates: %w", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if len(dups) <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
smpl := configmap.Simple{}
|
||||
smpl.Set("account", b.bbID)
|
||||
smpl.Set("key", b.bbKey)
|
||||
smpl.Set("chunk_size", strconv.FormatInt(int64(9600), 10))
|
||||
log.Println("duplicates", len(dups))
|
||||
for _, d := range dups {
|
||||
f, err := rcloneb2.NewFs(ctx, "B2", d.dir(), smpl)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rclonefs.NewFs %w", err)
|
||||
}
|
||||
if err := operations.CleanUp(ctx, f); err != nil {
|
||||
return fmt.Errorf("operations.CleanUp %w", err)
|
||||
}
|
||||
log.Println(d.dir(), "cleaned up")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package backblaze
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/kurin/blazer/b2"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type BackBlaze struct {
|
||||
logger *logrus.Logger
|
||||
bucketName string
|
||||
bbID string
|
||||
bbKey string
|
||||
dir string
|
||||
filePath string
|
||||
maxWorkers int
|
||||
b2Client *b2.Client
|
||||
}
|
||||
|
||||
// NewBackBlaze initializes a new BackBlaze struct with given BackBlaze ID and Key.
|
||||
func NewBackBlaze(ctx context.Context, logger *logrus.Logger, bbID, bbKey string) (*BackBlaze, error) {
|
||||
b2Client, err := b2.NewClient(ctx, bbID, bbKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("b2.NewClient %w", err)
|
||||
}
|
||||
|
||||
return &BackBlaze{
|
||||
logger: logger,
|
||||
b2Client: b2Client,
|
||||
bbID: bbID,
|
||||
bbKey: bbKey,
|
||||
maxWorkers: runtime.NumCPU(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *BackBlaze) WithBucket(bucketName string) *BackBlaze {
|
||||
b.bucketName = bucketName
|
||||
return b
|
||||
}
|
||||
func (b *BackBlaze) WithDir(dir string) *BackBlaze {
|
||||
b.dir = dir
|
||||
return b
|
||||
}
|
||||
func (b *BackBlaze) WithFile(filePath string) *BackBlaze {
|
||||
b.filePath = filePath
|
||||
return b
|
||||
}
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
package backblaze
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/kurin/blazer/b2"
|
||||
)
|
||||
|
||||
// localFiles lists the local files in the given backup directory and sends them to a channel.
|
||||
// It closes the channel after all files are listed.
|
||||
func (b *BackBlaze) localFiles(backupDir string, fileChan chan<- string) error {
|
||||
defer close(fileChan)
|
||||
// Walk the directory and send files to the channel
|
||||
err := filepath.WalkDir(backupDir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
fileChan <- filepath.Base(path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking the directory: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// b2BucketFiles lists the files in the given B2 bucket and sends them to a channel.
|
||||
// It closes the channel after all files are listed.
|
||||
func (b *BackBlaze) b2BucketFiles(ctx context.Context, bucketName string, fileChan chan<- string) error {
|
||||
bucket, err := b.b2Client.Bucket(ctx, bucketName)
|
||||
defer close(fileChan)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b2Client.Bucket %w", err)
|
||||
}
|
||||
|
||||
bucketIter := bucket.List(ctx, b2.ListHidden())
|
||||
if bucketIter == nil {
|
||||
return errors.New("bucket list cannot be nil")
|
||||
}
|
||||
|
||||
for {
|
||||
if !bucketIter.Next() {
|
||||
if bucketIter.Err() != nil {
|
||||
return fmt.Errorf("bucketIter err %w", bucketIter.Err())
|
||||
}
|
||||
break
|
||||
}
|
||||
if bucketIter.Object() == nil {
|
||||
return errors.New("bucketIter Object is nil")
|
||||
}
|
||||
b.logger.Debug("bucket file: ", bucketIter.Object().Name())
|
||||
fileChan <- bucketIter.Object().Name()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var ErrLocalNotInCloud error = errors.New("exists locally but not in the cloud")
|
||||
var ErrCloudNotInLocal error = errors.New("exists on cloud but not locally")
|
||||
|
||||
type B2LocalErr struct {
|
||||
File string
|
||||
Err error
|
||||
}
|
||||
|
||||
// CompareConcurrent concurrently fetches the list of local files and cloud files,
|
||||
// then compares them to ensure all local files exist in the cloud.
|
||||
// Errors are sent to a provided error channel. The function will panic if an error occurs while listing files.
|
||||
func (b *BackBlaze) CompareConcurrent(ctx context.Context, backupDir, bucketName string, errors chan<- B2LocalErr) {
|
||||
var wg sync.WaitGroup
|
||||
localFiles := make(map[string]int)
|
||||
cloudFiles := make(map[string]int)
|
||||
localFileChan := make(chan string)
|
||||
b2FileChan := make(chan string)
|
||||
|
||||
// Local listing
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for f := range localFileChan {
|
||||
if _, ok := localFiles[f]; ok {
|
||||
panic(fmt.Errorf("local file already exists in map: %s", f))
|
||||
}
|
||||
b.logger.Debug("local file", f)
|
||||
localFiles[f]++
|
||||
}
|
||||
}()
|
||||
|
||||
if err := b.localFiles(backupDir, localFileChan); err != nil {
|
||||
panic(fmt.Errorf("b.LocalFilesWithB2: %w", err))
|
||||
}
|
||||
}()
|
||||
|
||||
// Cloud listing
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for f := range b2FileChan {
|
||||
if _, ok := cloudFiles[f]; ok {
|
||||
panic(fmt.Errorf("cloud file already exists in map: %s", f))
|
||||
}
|
||||
b.logger.Debug("B2 file", f)
|
||||
cloudFiles[f]++
|
||||
}
|
||||
}()
|
||||
if err := b.b2BucketFiles(ctx, bucketName, b2FileChan); err != nil {
|
||||
panic(fmt.Errorf("b.LocalFilesWithB2: %w", err))
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for both to complete
|
||||
wg.Wait()
|
||||
|
||||
// Now check local files that are not presesnt in cloud
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for localFile := range localFiles {
|
||||
if _, exists := cloudFiles[localFile]; !exists {
|
||||
errors <- B2LocalErr{File: localFile, Err: ErrLocalNotInCloud}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Now check cloud files that are not in local
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for cloudFile := range cloudFiles {
|
||||
if _, exists := localFiles[cloudFile]; !exists {
|
||||
errors <- B2LocalErr{File: cloudFile, Err: ErrCloudNotInLocal}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
errors <- B2LocalErr{Err: nil}
|
||||
close(errors)
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package backblaze
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/rclone/rclone/backend/b2"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
)
|
||||
|
||||
func (b *BackBlaze) CleanUp(ctx context.Context, cancel context.CancelFunc, bucketName string) error {
|
||||
var dups []duplicate
|
||||
var err error
|
||||
if bucketName != "" {
|
||||
dups, err = b.listDuplicatesFromBucket(ctx, cancel, b.b2Client, bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b.listDuplicatesFromBucket: %w", err)
|
||||
}
|
||||
} else {
|
||||
dups, err = b.listDuplicates(ctx, cancel, b.b2Client)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b.listDuplicates: %w", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if len(dups) <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
smpl := configmap.Simple{}
|
||||
smpl.Set("account", b.bbID)
|
||||
smpl.Set("key", b.bbKey)
|
||||
smpl.Set("chunk_size", strconv.FormatInt(int64(9600), 10))
|
||||
b.logger.Info("duplicates", len(dups))
|
||||
for _, d := range dups {
|
||||
f, err := b2.NewFs(ctx, "B2", d.dir(), smpl)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rclonefs.NewFs %w", err)
|
||||
}
|
||||
if err := operations.CleanUp(ctx, f); err != nil {
|
||||
return fmt.Errorf("operations.CleanUp %w", err)
|
||||
}
|
||||
b.logger.Info(d.dir(), "cleaned up")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package services
|
||||
package backblaze
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
|
@ -12,6 +12,12 @@ import (
|
|||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
type duplicate struct {
|
||||
bucket string
|
||||
file string
|
||||
count int
|
||||
}
|
||||
|
||||
func (d duplicate) dir() string {
|
||||
if !strings.Contains(d.file, "/") {
|
||||
return d.bucket
|
||||
|
|
@ -20,7 +26,7 @@ func (d duplicate) dir() string {
|
|||
return strings.Join(splitted[:(len(splitted)-1)], "/")
|
||||
}
|
||||
|
||||
func (b *BackBalze) ListDuplicateVersions(ctx context.Context, cancel context.CancelFunc) error {
|
||||
func (b *BackBlaze) ListDuplicateVersions(ctx context.Context, cancel context.CancelFunc) error {
|
||||
b2Client, err := b2.NewClient(ctx, b.bbID, b.bbKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b2.NewClient %w", err)
|
||||
|
|
@ -42,7 +48,7 @@ func (b *BackBalze) ListDuplicateVersions(ctx context.Context, cancel context.Ca
|
|||
return nil
|
||||
}
|
||||
|
||||
func (b *BackBalze) listDuplicates(ctx context.Context, cancel context.CancelFunc, b2Client *b2.Client) ([]duplicate, error) {
|
||||
func (b *BackBlaze) listDuplicates(ctx context.Context, cancel context.CancelFunc, b2Client *b2.Client) ([]duplicate, error) {
|
||||
buckets, err := b2Client.ListBuckets(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("b2Client.Bucket %w", err)
|
||||
|
|
@ -66,20 +72,20 @@ func (b *BackBalze) listDuplicates(ctx context.Context, cancel context.CancelFun
|
|||
|
||||
bucketIter := bc.List(ctx, b2.ListHidden())
|
||||
if bucketIter == nil {
|
||||
log.Println("bucket list cannot be nil")
|
||||
b.logger.Error("bucket list cannot be nil")
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
if !bucketIter.Next() {
|
||||
if bucketIter.Err() != nil {
|
||||
log.Println("bucketIter err %w", bucketIter.Err())
|
||||
b.logger.Error("bucketIter err %w", bucketIter.Err())
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
if bucketIter.Object() == nil {
|
||||
log.Println("bucketIter Object is nil")
|
||||
b.logger.Error("bucketIter Object is nil")
|
||||
continue
|
||||
}
|
||||
files[bucketIter.Object().Name()]++
|
||||
|
|
@ -102,7 +108,7 @@ func (b *BackBalze) listDuplicates(ctx context.Context, cancel context.CancelFun
|
|||
return dups, nil
|
||||
}
|
||||
|
||||
func (b *BackBalze) listDuplicatesFromBucket(ctx context.Context, cancel context.CancelFunc, b2Client *b2.Client, bucketName string) ([]duplicate, error) {
|
||||
func (b *BackBlaze) listDuplicatesFromBucket(ctx context.Context, cancel context.CancelFunc, b2Client *b2.Client, bucketName string) ([]duplicate, error) {
|
||||
bucket, err := b2Client.Bucket(ctx, bucketName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("b2Client.Bucket %w", err)
|
||||
|
|
@ -0,0 +1,169 @@
|
|||
package backblaze
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/kurin/blazer/b2"
|
||||
)
|
||||
|
||||
const writers = 4
|
||||
|
||||
func (b *BackBlaze) Sync(ctx context.Context) error {
|
||||
if b.bucketName == "" && (b.filePath == "" || b.dir == "") {
|
||||
return fmt.Errorf("bucket name is %v | filePath is %v | dir is %v", b.bucketName, b.filePath, b.dir)
|
||||
}
|
||||
|
||||
if b.filePath != "" && b.dir != "" {
|
||||
return errors.New("you must select just 1 option, dir or file")
|
||||
}
|
||||
|
||||
bc, err := b.b2Client.Bucket(ctx, b.bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("b2Client.Bucket %w", err)
|
||||
}
|
||||
|
||||
if bc == nil {
|
||||
return fmt.Errorf("bucket doesn't exist %s", b.bucketName)
|
||||
}
|
||||
|
||||
log.Println("bucket found:", bc.Name())
|
||||
if b.filePath != "" {
|
||||
log.Println("file:", b.filePath)
|
||||
|
||||
if err := b.copyFile(ctx, bc, b.filePath); err != nil {
|
||||
return fmt.Errorf("copyFile %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if b.dir != "" {
|
||||
oldFiles, err := b.bucketFiles(ctx, bc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bucketFiles %w", err)
|
||||
}
|
||||
b.logger.Debug(strings.Repeat("*", 40))
|
||||
b.logger.Debug("oldFiles to clean:\n\t\t" + strings.Join(oldFiles, "\n\t\t"))
|
||||
b.logger.Debug(strings.Repeat("*", 40))
|
||||
|
||||
fileChan := make(chan string)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := 0; i < b.maxWorkers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for src := range fileChan {
|
||||
if err := b.copyFile(ctx, bc, src); err != nil {
|
||||
b.logger.Errorf("error copying file %s: %v\n", src, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Walk the directory and send files to the channel for uploading
|
||||
err = filepath.WalkDir(b.dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
fileChan <- path
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking the directory: %v", err)
|
||||
}
|
||||
|
||||
// Close the channel (no more files to send)
|
||||
close(fileChan)
|
||||
wg.Wait()
|
||||
|
||||
// Cleanup old files after backup is completed
|
||||
if err := b.cleanBucket(ctx, bc, oldFiles); err != nil {
|
||||
return fmt.Errorf("cleanBucket %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
b.logger.Info("copied successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BackBlaze) copyFile(ctx context.Context, bucket *b2.Bucket, src string) error {
|
||||
f, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w := bucket.Object(fi.Name()).NewWriter(ctx)
|
||||
w.ConcurrentUploads = writers
|
||||
w.UseFileBuffer = true
|
||||
b.logger.Info("start copying", fi.Name())
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
w.Close()
|
||||
return err
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.logger.Info("end copying", fi.Name())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BackBlaze) cleanBucket(ctx context.Context, bucket *b2.Bucket, files []string) error {
|
||||
var errorBuilder strings.Builder
|
||||
for _, v := range files {
|
||||
obj := bucket.Object(v)
|
||||
if obj == nil {
|
||||
b.logger.Error("bucket.Object is nil", v)
|
||||
continue
|
||||
}
|
||||
if err := obj.Delete(ctx); err != nil {
|
||||
errorBuilder.WriteString(fmt.Errorf("error deleting %s : %w", v, err).Error())
|
||||
errorBuilder.WriteString("; ")
|
||||
}
|
||||
}
|
||||
|
||||
if errorBuilder.Len() > 0 {
|
||||
return errors.New(errorBuilder.String())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BackBlaze) bucketFiles(ctx context.Context, bucket *b2.Bucket) ([]string, error) {
|
||||
bucketIter := bucket.List(ctx)
|
||||
if bucketIter == nil {
|
||||
return nil, fmt.Errorf("bucket list cannot be nil")
|
||||
}
|
||||
var files []string
|
||||
for {
|
||||
if !bucketIter.Next() {
|
||||
if bucketIter.Err() != nil {
|
||||
return nil, fmt.Errorf("bucketIter err %w", bucketIter.Err())
|
||||
}
|
||||
break
|
||||
}
|
||||
if bucketIter.Object() == nil {
|
||||
b.logger.Error("bucketIter Object is nil")
|
||||
continue
|
||||
}
|
||||
files = append(files, bucketIter.Object().Name())
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,183 @@
|
|||
package email
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/smtp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
mime = "MIME-version: 1.0;\nContent-Type: text/html; charset=\"UTF-8\";\n\n"
|
||||
subjectReport = "B2 vs Local Files"
|
||||
templateSuccess = "success.html"
|
||||
templateConfirm = "confirm.html"
|
||||
templateForgotPassword = "forgot_password.html"
|
||||
templateRegister = "register.html"
|
||||
delimeter = "**=myohmy689407924327"
|
||||
)
|
||||
|
||||
type EmailService struct {
|
||||
auth smtp.Auth
|
||||
host string
|
||||
port string
|
||||
from string
|
||||
tlsconfig *tls.Config
|
||||
}
|
||||
|
||||
type EmailWithAttachments struct {
|
||||
To string
|
||||
Bucket string
|
||||
BackupDir string
|
||||
CountLocal int
|
||||
CountCloud int
|
||||
Attachments []EmailAttachment
|
||||
}
|
||||
|
||||
type EmailAttachment struct {
|
||||
File io.Reader
|
||||
Title string
|
||||
}
|
||||
|
||||
func (e EmailAttachment) ReadContent() ([]byte, error) {
|
||||
bts, err := io.ReadAll(e.File)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error loading attachment: %s", err)
|
||||
}
|
||||
return bts, nil
|
||||
}
|
||||
|
||||
type MailServiceConfig struct {
|
||||
Auth smtp.Auth
|
||||
Host string
|
||||
Port string
|
||||
From string // Sender email address
|
||||
}
|
||||
|
||||
func NewMailService(config MailServiceConfig) *EmailService {
|
||||
return &EmailService{
|
||||
auth: config.Auth,
|
||||
host: config.Host,
|
||||
port: config.Port,
|
||||
from: config.From,
|
||||
tlsconfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
ServerName: config.Host,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *EmailService) SendOK(emailData EmailWithAttachments) error {
|
||||
template := strings.Replace(htmlTemplate, "{{bucket}}", emailData.Bucket, -1)
|
||||
template = strings.Replace(template, "{{local_backup_path}}", emailData.BackupDir, -1)
|
||||
template = strings.Replace(template, "{{count_ErrCloudNotInLocal}}", fmt.Sprint(emailData.CountCloud), -1)
|
||||
template = strings.Replace(template, "{{count_ErrLocalNotInCloud}}", fmt.Sprint(emailData.CountLocal), -1)
|
||||
|
||||
msg, err := newMessage(e.from, emailData.To, subjectReport).
|
||||
withAttachments(template, emailData.Attachments)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("error while upload attachments %w", err)
|
||||
}
|
||||
|
||||
return e.send(emailData.To, msg)
|
||||
}
|
||||
|
||||
func (e *EmailService) send(to string, msg []byte) error {
|
||||
c, err := smtp.Dial(e.host + ":" + e.port)
|
||||
if err != nil {
|
||||
return fmt.Errorf("DIAL: %s", err)
|
||||
}
|
||||
|
||||
if err = c.StartTLS(e.tlsconfig); err != nil {
|
||||
return fmt.Errorf("c.StartTLS: %s", err)
|
||||
}
|
||||
|
||||
// Auth
|
||||
if err = c.Auth(e.auth); err != nil {
|
||||
return fmt.Errorf("c.Auth: %s", err)
|
||||
}
|
||||
|
||||
// To && From
|
||||
if err = c.Mail(e.from); err != nil {
|
||||
return fmt.Errorf("c.Mail: %s", err)
|
||||
}
|
||||
|
||||
if err = c.Rcpt(to); err != nil {
|
||||
return fmt.Errorf("c.Rcpt: %s", err)
|
||||
}
|
||||
|
||||
// Data
|
||||
w, err := c.Data()
|
||||
if err != nil {
|
||||
return fmt.Errorf("c.Data: %s", err)
|
||||
}
|
||||
|
||||
written, err := w.Write(msg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("w.Write: %s", err)
|
||||
}
|
||||
|
||||
if written <= 0 {
|
||||
return fmt.Errorf("%d bytes written", written)
|
||||
}
|
||||
|
||||
if err = w.Close(); err != nil {
|
||||
return fmt.Errorf("w.Close: %s", err)
|
||||
}
|
||||
|
||||
if err = c.Quit(); err != nil {
|
||||
return fmt.Errorf("w.Quit: %s", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type message struct {
|
||||
from string
|
||||
to string
|
||||
subject string
|
||||
}
|
||||
|
||||
func newMessage(from, to, subject string) message {
|
||||
return message{from: from, to: to, subject: subject}
|
||||
}
|
||||
|
||||
func (m message) withAttachments(body string, attachments []EmailAttachment) ([]byte, error) {
|
||||
headers := make(map[string]string)
|
||||
headers["From"] = m.from
|
||||
headers["To"] = m.to
|
||||
headers["Subject"] = m.subject
|
||||
headers["MIME-Version"] = "1.0"
|
||||
|
||||
var message bytes.Buffer
|
||||
|
||||
for k, v := range headers {
|
||||
message.WriteString(k)
|
||||
message.WriteString(": ")
|
||||
message.WriteString(v)
|
||||
message.WriteString("\r\n")
|
||||
}
|
||||
|
||||
message.WriteString("Content-Type: " + fmt.Sprintf("multipart/mixed; boundary=\"%s\"\r\n", delimeter))
|
||||
message.WriteString("--" + delimeter + "\r\n")
|
||||
message.WriteString("Content-Type: text/html; charset=\"UTF-8\"\r\n\r\n")
|
||||
message.WriteString(body + "\r\n\r\n")
|
||||
|
||||
for _, attachment := range attachments {
|
||||
attachmentRawFile, err := attachment.ReadContent()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
message.WriteString("--" + delimeter + "\r\n")
|
||||
message.WriteString("Content-Disposition: attachment; filename=\"" + attachment.Title + "\"\r\n")
|
||||
message.WriteString("Content-Type: application/octet-stream\r\n")
|
||||
message.WriteString("Content-Transfer-Encoding: base64\r\n\r\n")
|
||||
message.WriteString(base64.StdEncoding.EncodeToString(attachmentRawFile) + "\r\n")
|
||||
}
|
||||
|
||||
message.WriteString("--" + delimeter + "--") // End the message
|
||||
return message.Bytes(), nil
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
package email
|
||||
|
||||
import (
|
||||
"net/smtp"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"gitea.urkob.com/urko/backblaze-backup/kit"
|
||||
"gitea.urkob.com/urko/backblaze-backup/kit/config"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_mailService_SendOK(t *testing.T) {
|
||||
cfg := config.NewConfig(kit.RootDir() + "/.env.test")
|
||||
|
||||
mailSrv := NewMailService(MailServiceConfig{
|
||||
Auth: smtp.PlainAuth("", cfg.MailUser, cfg.MailPassword, cfg.MailHost),
|
||||
Host: cfg.MailHost,
|
||||
Port: cfg.MailPort,
|
||||
From: cfg.MailUser,
|
||||
},
|
||||
)
|
||||
reader, err := os.Open("testdata/attachment1.txt")
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
|
||||
reader2, err := os.Open("testdata/attachment2.txt")
|
||||
require.NoError(t, err)
|
||||
defer reader2.Close()
|
||||
|
||||
reader3, err := os.Open("testdata/attachment3.txt")
|
||||
require.NoError(t, err)
|
||||
defer reader3.Close()
|
||||
|
||||
data := EmailWithAttachments{
|
||||
To: cfg.MailTo,
|
||||
Attachments: []EmailAttachment{
|
||||
{
|
||||
Title: "attachment1.txt",
|
||||
File: reader,
|
||||
},
|
||||
{
|
||||
Title: "attachment2.txt",
|
||||
File: reader2,
|
||||
},
|
||||
{
|
||||
Title: "attachment3.txt",
|
||||
File: reader3,
|
||||
},
|
||||
},
|
||||
}
|
||||
err = mailSrv.SendOK(data)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package email
|
||||
|
||||
const htmlTemplate = `<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Report</title>
|
||||
</head>
|
||||
<body style="font-family: Arial, sans-serif; margin: 0; padding: 0;">
|
||||
<div style="background-color: #f4f4f4; padding: 20px;">
|
||||
<div style="max-width: 600px; background-color: #ffffff; margin: 0 auto; padding: 20px; border-radius: 4px; box-shadow: 0 0 10px rgba(0,0,0,0.1);">
|
||||
<h2 style="color: #333333; margin-top: 0;">Bucket Report</h2>
|
||||
<hr style="border: none; border-bottom: 1px solid #ddd;">
|
||||
<p><strong>Local Backup Path:</strong> {{local_backup_path}}</p>
|
||||
<p><strong>Bucket Name:</strong> {{bucket}}</p>
|
||||
<p><strong>Count of ErrCloudNotInLocal:</strong> {{count_ErrCloudNotInLocal}}</p>
|
||||
<p><strong>Count of ErrLocalNotInCloud:</strong> {{count_ErrLocalNotInCloud}}</p>
|
||||
<hr style="border: none; border-bottom: 1px solid #ddd;">
|
||||
<p style="text-align: center; color: #666666;">This is an automated report, please do not reply to this email.</p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
|
|
@ -0,0 +1 @@
|
|||
this is txt 1 attachment
|
||||
|
|
@ -0,0 +1 @@
|
|||
this is txt 2 attachment
|
||||
|
|
@ -0,0 +1 @@
|
|||
this is txt 3 attachment
|
||||
|
|
@ -1,30 +1,36 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"log"
|
||||
"fmt"
|
||||
|
||||
"gitea.urkob.com/urko/backblaze-backup/kit"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/kelseyhightower/envconfig"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
BbId string `required:"true" split_words:"true"`
|
||||
BbKey string `required:"false" split_words:"true"`
|
||||
BbId string `required:"true" split_words:"true"`
|
||||
BbKey string `required:"true" split_words:"true"`
|
||||
LogLevel uint32 `required:"false" split_words:"true" default:"4"`
|
||||
// Mail vars
|
||||
MailTo string `required:"true" split_words:"true"`
|
||||
MailHost string `required:"true" split_words:"true"`
|
||||
MailPort string `required:"true" split_words:"true"`
|
||||
MailUser string `required:"true" split_words:"true"`
|
||||
MailPassword string `required:"true" split_words:"true"`
|
||||
}
|
||||
|
||||
func NewConfig(envFile string) *Config {
|
||||
if envFile != "" {
|
||||
err := godotenv.Load(kit.RootDir() + "/" + envFile)
|
||||
err := godotenv.Load(envFile)
|
||||
if err != nil {
|
||||
log.Fatalln("godotenv.Load:", err)
|
||||
panic(fmt.Errorf("godotenv.Load: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
cfg := &Config{}
|
||||
err := envconfig.Process("", cfg)
|
||||
if err != nil {
|
||||
log.Fatalf("envconfig.Process: %s\n", err)
|
||||
panic(fmt.Errorf("envconfig.Process: %w", err))
|
||||
}
|
||||
|
||||
return cfg
|
||||
|
|
|
|||
7
main.go
7
main.go
|
|
@ -15,16 +15,19 @@ var rootCmd = &cobra.Command{
|
|||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(cmd.Sync, cmd.Versions, cmd.Cleanup)
|
||||
rootCmd.AddCommand(cmd.Sync, cmd.Versions, cmd.Cleanup, cmd.Check)
|
||||
cmd.Sync.PersistentFlags().String("file", "", "absolute path of the file you want to upload to backblaze")
|
||||
cmd.Sync.PersistentFlags().String("dir", "", "absolute path of the directory you want to upload to backblaze")
|
||||
cmd.Sync.PersistentFlags().String("bucket", "", "backblaze bucket name")
|
||||
|
||||
cmd.Check.PersistentFlags().String("dir", "", "Specifies the absolute path of the directory containing the backup files to be compared against the Backblaze B2 bucket. This flag is mutually exclusive with the 'file' flag.")
|
||||
cmd.Check.PersistentFlags().String("bucket", "", "Name of the Backblaze B2 bucket against which the local files or directory will be compared.")
|
||||
|
||||
cmd.Cleanup.PersistentFlags().String("bucket", "", "backblaze bucket name")
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := cmd.Sync.Execute(); err != nil {
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue