fixed rsync command for files with special character (fu***ing backticks)

This commit is contained in:
Daniel Sommer 2023-10-13 13:00:32 +02:00
parent c089eb84bd
commit 2371ed3318
3 changed files with 45 additions and 17 deletions

6
.vscode/launch.json vendored
View file

@ -12,10 +12,10 @@
"--password", "--password",
"$Velvet90", "$Velvet90",
"--concurrency", "--concurrency",
"12", "24",
"--delay", "--delay",
"250", "100",
"/home/velvettear/downloads/music/*", "/mnt/kingston/downloads/music/*",
"192.168.1.11:/share/tmp/music", "192.168.1.11:/share/tmp/music",
], ],
"console": "integratedTerminal" "console": "integratedTerminal"

View file

@ -73,7 +73,6 @@ func trace(level int, timestamp int64, message string, extras ...string) {
message += " (" + suffix + ")" message += " (" + suffix + ")"
} }
if timestamp >= 0 { if timestamp >= 0 {
message += " [" + strconv.Itoa(int(time.Now().UnixMilli()-timestamp)) + "ms" + "]" message += " [" + strconv.Itoa(int(time.Now().UnixMilli()-timestamp)) + "ms" + "]"
} }
fmt.Println(buildLogMessage(getPrefixForLogLevel(level), message)) fmt.Println(buildLogMessage(getPrefixForLogLevel(level), message))

View file

@ -1,6 +1,7 @@
package tools package tools
import ( import (
"errors"
"io" "io"
"os" "os"
"os/exec" "os/exec"
@ -37,13 +38,14 @@ func Transfer() {
mpb.WithWaitGroup(&waitgroup), mpb.WithWaitGroup(&waitgroup),
) )
timestamp := time.Now() timestamp := time.Now()
counter := 0
totalbar := createProgressBar(barcontainer, strconv.FormatFloat(sourceFileSize, 'f', 2, 64)+sourceFileSizeName+" 󰇙 Total", int64(0), int64(sourcefilesCount), sourcefilesCount+1, true) totalbar := createProgressBar(barcontainer, strconv.FormatFloat(sourceFileSize, 'f', 2, 64)+sourceFileSizeName+" 󰇙 Total", int64(0), int64(sourcefilesCount), sourcefilesCount+1, true)
concurrency := settings.Concurrency concurrency := settings.Concurrency
if concurrency == 0 { if concurrency == 0 {
concurrency = sourcefilesCount concurrency = sourcefilesCount
} }
counter := 0
index := 0 index := 0
errors := make(map[string]error)
channel := make(chan struct{}, concurrency) channel := make(chan struct{}, concurrency)
for file, size := range sourcefiles { for file, size := range sourcefiles {
channel <- struct{}{} channel <- struct{}{}
@ -53,7 +55,12 @@ func Transfer() {
go func(file string, size int64, index int) { go func(file string, size int64, index int) {
defer waitgroup.Done() defer waitgroup.Done()
bar := createProgressBar(barcontainer, filepath.Base(file), size, int64(100), index, false) bar := createProgressBar(barcontainer, filepath.Base(file), size, int64(100), index, false)
if transferFile(bar, file) { error := transferFile(bar, file)
if error != nil {
errors[file] = error
bar.Abort(false)
bar.Wait()
} else {
counter++ counter++
} }
totalbar.Increment() totalbar.Increment()
@ -69,10 +76,16 @@ func Transfer() {
transferSpeedName += "/s" transferSpeedName += "/s"
transferSize, transferSizeName := humanizeBytes(transferSize) transferSize, transferSizeName := humanizeBytes(transferSize)
log.InfoTimed("transferred "+strconv.Itoa(counter)+" files, "+strconv.Itoa(int(transferSize))+" "+transferSizeName+" ("+strconv.FormatFloat(transferSpeed, 'f', 2, 64)+" "+transferSpeedName+")", timestamp.UnixMilli()) log.InfoTimed("transferred "+strconv.Itoa(counter)+" files, "+strconv.Itoa(int(transferSize))+" "+transferSizeName+" ("+strconv.FormatFloat(transferSpeed, 'f', 2, 64)+" "+transferSpeedName+")", timestamp.UnixMilli())
if len(errors) > 0 {
log.Info("encountered " + strconv.Itoa(len(errors)) + " errors")
for file, error := range errors {
log.Info(file, error.Error())
}
}
} }
// unexported function(s) // unexported function(s)
func transferFile(bar *mpb.Bar, file string) bool { func transferFile(bar *mpb.Bar, file string) error {
target := getTargetLocation(file) target := getTargetLocation(file)
var arguments []string var arguments []string
if len(settings.Password) > 0 { if len(settings.Password) > 0 {
@ -89,16 +102,18 @@ func transferFile(bar *mpb.Bar, file string) bool {
target = settings.User + "@" + target target = settings.User + "@" + target
} }
} }
arguments = append(arguments, "rsync", "-avz", "--mkpath", file, target, "--progress") arguments = append(arguments, "rsync", "-avz", "--secluded-args", "--mkpath", file, target, "--progress")
cmd := exec.Command("sshpass", arguments...) cmd := exec.Command("sshpass", arguments...)
stdout, stdoutError := cmd.StdoutPipe() stdout, stdoutError := cmd.StdoutPipe()
stderr, stderrError := cmd.StderrPipe() stderr, stderrError := cmd.StderrPipe()
cmd.Start() cmd.Start()
if stdoutError != nil { if stdoutError != nil {
log.Fatal(stdoutError.Error()) log.Warning("encountered an error opening remote stdout pipe", "file: "+file, stdoutError.Error())
return stderrError
} }
if stderrError != nil { if stderrError != nil {
log.Fatal(stderrError.Error()) log.Warning("encountered an error opening remote stderr pipe", "file: "+file, stdoutError.Error())
return stderrError
} }
var resultBytes []byte var resultBytes []byte
var waitgroup sync.WaitGroup var waitgroup sync.WaitGroup
@ -110,7 +125,7 @@ func transferFile(bar *mpb.Bar, file string) bool {
readBytes, error := stdout.Read(tmp) readBytes, error := stdout.Read(tmp)
if error != nil { if error != nil {
if error != io.EOF { if error != io.EOF {
log.Warning("encountered an error reading stdout", error.Error()) log.Warning("encountered an error reading stdout", "file: "+file, error.Error())
} }
break break
} }
@ -126,7 +141,7 @@ func transferFile(bar *mpb.Bar, file string) bool {
tmp = strings.ReplaceAll(strings.TrimSpace(tmp), ".", "") tmp = strings.ReplaceAll(strings.TrimSpace(tmp), ".", "")
bytes, error := strconv.ParseInt(tmp, 10, 64) bytes, error := strconv.ParseInt(tmp, 10, 64)
if error != nil { if error != nil {
log.Fatal("encountered an error converting the transferred bytes to int", error.Error()) log.Fatal("encountered an error converting the transferred bytes to int", "file: "+file, error.Error())
} }
transferSize += bytes transferSize += bytes
} }
@ -156,17 +171,19 @@ func transferFile(bar *mpb.Bar, file string) bool {
}() }()
errorBytes, stderrError := io.ReadAll(stderr) errorBytes, stderrError := io.ReadAll(stderr)
if stderrError != nil { if stderrError != nil {
log.Fatal(stderrError.Error()) log.Warning("encountered an error reading from remote stderr", "file: "+file, stdoutError.Error())
return stderrError
} }
cmd.Wait() cmd.Wait()
error := strings.Trim(string(errorBytes), "\n") error := strings.Trim(string(errorBytes), "\n")
if len(error) > 0 { if len(error) > 0 {
log.Fatal(error) log.Warning("encountered an remote error", "file: "+file, error)
return errors.New(error)
} }
waitgroup.Wait() waitgroup.Wait()
stdout.Close() stdout.Close()
stderr.Close() stderr.Close()
return true return nil
} }
func getTargetLocation(sourceFile string) string { func getTargetLocation(sourceFile string) string {
@ -182,16 +199,22 @@ func getTargetLocation(sourceFile string) string {
} }
func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max int64, priority int, total bool) *mpb.Bar { func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max int64, priority int, total bool) *mpb.Bar {
red, green, magenta, yellow := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow) _, green, magenta, yellow, blue := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow), color.New(color.FgBlue)
barstyle := mpb.BarStyle().Lbound("").Filler("󰝤").Tip("").Padding(" ").Rbound("") barstyle := mpb.BarStyle().Lbound("").Filler("󰝤").Tip("").Padding(" ").Rbound("")
defaultBarPrepend := mpb.PrependDecorators( defaultBarPrepend := mpb.PrependDecorators(
decor.Name("[info] > "), decor.Name("[info] > "),
decor.OnCompleteMeta( decor.OnCompleteMeta(
decor.OnComplete( decor.OnComplete(
decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(red)), ""+name, decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(blue)), ""+name,
), ),
colorize(green), colorize(green),
), ),
// decor.OnAbortMeta(
// decor.OnAbort(
// decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(blue)), ""+name,
// ),
// colorize(red),
// ),
) )
defaultBarAppend := mpb.AppendDecorators( defaultBarAppend := mpb.AppendDecorators(
decor.OnCompleteMeta(decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpaceR), colorize(yellow)), decor.OnCompleteMeta(decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpaceR), colorize(yellow)),
@ -201,6 +224,12 @@ func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max
decor.OnComplete( decor.OnComplete(
decor.Percentage(decor.WCSyncSpaceR), "", decor.Percentage(decor.WCSyncSpaceR), "",
), ),
// decor.OnAbort(
// decor.Name("󰇙", decor.WCSyncSpaceR), "",
// ),
// decor.OnAbort(
// decor.Percentage(decor.WCSyncSpace), "",
// ),
) )
totalBarPrepend := mpb.PrependDecorators( totalBarPrepend := mpb.PrependDecorators(
decor.Name("[info] > "), decor.Name("[info] > "),