From 2371ed3318cdd0a786ce093ff2fd0cd0467b1577 Mon Sep 17 00:00:00 2001 From: velvettear Date: Fri, 13 Oct 2023 13:00:32 +0200 Subject: [PATCH] fixed rsync command for files with special character (fu***ing backticks) --- .vscode/launch.json | 6 ++--- log/log.go | 1 - tools/rsync.go | 55 ++++++++++++++++++++++++++++++++++----------- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index aaea6bc..a8e747c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,10 +12,10 @@ "--password", "$Velvet90", "--concurrency", - "12", + "24", "--delay", - "250", - "/home/velvettear/downloads/music/*", + "100", + "/mnt/kingston/downloads/music/*", "192.168.1.11:/share/tmp/music", ], "console": "integratedTerminal" diff --git a/log/log.go b/log/log.go index e4ff346..e6bf124 100644 --- a/log/log.go +++ b/log/log.go @@ -73,7 +73,6 @@ func trace(level int, timestamp int64, message string, extras ...string) { message += " (" + suffix + ")" } if timestamp >= 0 { - message += " [" + strconv.Itoa(int(time.Now().UnixMilli()-timestamp)) + "ms" + "]" } fmt.Println(buildLogMessage(getPrefixForLogLevel(level), message)) diff --git a/tools/rsync.go b/tools/rsync.go index 425cd7d..790c343 100644 --- a/tools/rsync.go +++ b/tools/rsync.go @@ -1,6 +1,7 @@ package tools import ( + "errors" "io" "os" "os/exec" @@ -37,13 +38,14 @@ func Transfer() { mpb.WithWaitGroup(&waitgroup), ) timestamp := time.Now() - counter := 0 totalbar := createProgressBar(barcontainer, strconv.FormatFloat(sourceFileSize, 'f', 2, 64)+sourceFileSizeName+" 󰇙 Total", int64(0), int64(sourcefilesCount), sourcefilesCount+1, true) concurrency := settings.Concurrency if concurrency == 0 { concurrency = sourcefilesCount } + counter := 0 index := 0 + errors := make(map[string]error) channel := make(chan struct{}, concurrency) for file, size := range sourcefiles { channel <- struct{}{} @@ -53,7 +55,12 @@ func Transfer() { go func(file string, size int64, index int) { defer waitgroup.Done() bar := createProgressBar(barcontainer, filepath.Base(file), size, int64(100), index, false) - if transferFile(bar, file) { + error := transferFile(bar, file) + if error != nil { + errors[file] = error + bar.Abort(false) + bar.Wait() + } else { counter++ } totalbar.Increment() @@ -69,10 +76,16 @@ func Transfer() { transferSpeedName += "/s" transferSize, transferSizeName := humanizeBytes(transferSize) log.InfoTimed("transferred "+strconv.Itoa(counter)+" files, "+strconv.Itoa(int(transferSize))+" "+transferSizeName+" ("+strconv.FormatFloat(transferSpeed, 'f', 2, 64)+" "+transferSpeedName+")", timestamp.UnixMilli()) + if len(errors) > 0 { + log.Info("encountered " + strconv.Itoa(len(errors)) + " errors") + for file, error := range errors { + log.Info(file, error.Error()) + } + } } // unexported function(s) -func transferFile(bar *mpb.Bar, file string) bool { +func transferFile(bar *mpb.Bar, file string) error { target := getTargetLocation(file) var arguments []string if len(settings.Password) > 0 { @@ -89,16 +102,18 @@ func transferFile(bar *mpb.Bar, file string) bool { target = settings.User + "@" + target } } - arguments = append(arguments, "rsync", "-avz", "--mkpath", file, target, "--progress") + arguments = append(arguments, "rsync", "-avz", "--secluded-args", "--mkpath", file, target, "--progress") cmd := exec.Command("sshpass", arguments...) stdout, stdoutError := cmd.StdoutPipe() stderr, stderrError := cmd.StderrPipe() cmd.Start() if stdoutError != nil { - log.Fatal(stdoutError.Error()) + log.Warning("encountered an error opening remote stdout pipe", "file: "+file, stdoutError.Error()) + return stderrError } if stderrError != nil { - log.Fatal(stderrError.Error()) + log.Warning("encountered an error opening remote stderr pipe", "file: "+file, stdoutError.Error()) + return stderrError } var resultBytes []byte var waitgroup sync.WaitGroup @@ -110,7 +125,7 @@ func transferFile(bar *mpb.Bar, file string) bool { readBytes, error := stdout.Read(tmp) if error != nil { if error != io.EOF { - log.Warning("encountered an error reading stdout", error.Error()) + log.Warning("encountered an error reading stdout", "file: "+file, error.Error()) } break } @@ -126,7 +141,7 @@ func transferFile(bar *mpb.Bar, file string) bool { tmp = strings.ReplaceAll(strings.TrimSpace(tmp), ".", "") bytes, error := strconv.ParseInt(tmp, 10, 64) if error != nil { - log.Fatal("encountered an error converting the transferred bytes to int", error.Error()) + log.Fatal("encountered an error converting the transferred bytes to int", "file: "+file, error.Error()) } transferSize += bytes } @@ -156,17 +171,19 @@ func transferFile(bar *mpb.Bar, file string) bool { }() errorBytes, stderrError := io.ReadAll(stderr) if stderrError != nil { - log.Fatal(stderrError.Error()) + log.Warning("encountered an error reading from remote stderr", "file: "+file, stdoutError.Error()) + return stderrError } cmd.Wait() error := strings.Trim(string(errorBytes), "\n") if len(error) > 0 { - log.Fatal(error) + log.Warning("encountered an remote error", "file: "+file, error) + return errors.New(error) } waitgroup.Wait() stdout.Close() stderr.Close() - return true + return nil } func getTargetLocation(sourceFile string) string { @@ -182,16 +199,22 @@ func getTargetLocation(sourceFile string) string { } func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max int64, priority int, total bool) *mpb.Bar { - red, green, magenta, yellow := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow) + _, green, magenta, yellow, blue := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow), color.New(color.FgBlue) barstyle := mpb.BarStyle().Lbound("").Filler("󰝤").Tip("").Padding(" ").Rbound("") defaultBarPrepend := mpb.PrependDecorators( decor.Name("[info] > "), decor.OnCompleteMeta( decor.OnComplete( - decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(red)), ""+name, + decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(blue)), ""+name, ), colorize(green), ), + // decor.OnAbortMeta( + // decor.OnAbort( + // decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(blue)), ""+name, + // ), + // colorize(red), + // ), ) defaultBarAppend := mpb.AppendDecorators( decor.OnCompleteMeta(decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpaceR), colorize(yellow)), @@ -201,6 +224,12 @@ func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max decor.OnComplete( decor.Percentage(decor.WCSyncSpaceR), "", ), + // decor.OnAbort( + // decor.Name("󰇙", decor.WCSyncSpaceR), "", + // ), + // decor.OnAbort( + // decor.Percentage(decor.WCSyncSpace), "", + // ), ) totalBarPrepend := mpb.PrependDecorators( decor.Name("[info] > "),