fixed rsync command for files with special character (fu***ing backticks)
This commit is contained in:
parent
c089eb84bd
commit
2371ed3318
3 changed files with 45 additions and 17 deletions
6
.vscode/launch.json
vendored
6
.vscode/launch.json
vendored
|
@ -12,10 +12,10 @@
|
||||||
"--password",
|
"--password",
|
||||||
"$Velvet90",
|
"$Velvet90",
|
||||||
"--concurrency",
|
"--concurrency",
|
||||||
"12",
|
"24",
|
||||||
"--delay",
|
"--delay",
|
||||||
"250",
|
"100",
|
||||||
"/home/velvettear/downloads/music/*",
|
"/mnt/kingston/downloads/music/*",
|
||||||
"192.168.1.11:/share/tmp/music",
|
"192.168.1.11:/share/tmp/music",
|
||||||
],
|
],
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
|
|
|
@ -73,7 +73,6 @@ func trace(level int, timestamp int64, message string, extras ...string) {
|
||||||
message += " (" + suffix + ")"
|
message += " (" + suffix + ")"
|
||||||
}
|
}
|
||||||
if timestamp >= 0 {
|
if timestamp >= 0 {
|
||||||
|
|
||||||
message += " [" + strconv.Itoa(int(time.Now().UnixMilli()-timestamp)) + "ms" + "]"
|
message += " [" + strconv.Itoa(int(time.Now().UnixMilli()-timestamp)) + "ms" + "]"
|
||||||
}
|
}
|
||||||
fmt.Println(buildLogMessage(getPrefixForLogLevel(level), message))
|
fmt.Println(buildLogMessage(getPrefixForLogLevel(level), message))
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package tools
|
package tools
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
@ -37,13 +38,14 @@ func Transfer() {
|
||||||
mpb.WithWaitGroup(&waitgroup),
|
mpb.WithWaitGroup(&waitgroup),
|
||||||
)
|
)
|
||||||
timestamp := time.Now()
|
timestamp := time.Now()
|
||||||
counter := 0
|
|
||||||
totalbar := createProgressBar(barcontainer, strconv.FormatFloat(sourceFileSize, 'f', 2, 64)+sourceFileSizeName+" Total", int64(0), int64(sourcefilesCount), sourcefilesCount+1, true)
|
totalbar := createProgressBar(barcontainer, strconv.FormatFloat(sourceFileSize, 'f', 2, 64)+sourceFileSizeName+" Total", int64(0), int64(sourcefilesCount), sourcefilesCount+1, true)
|
||||||
concurrency := settings.Concurrency
|
concurrency := settings.Concurrency
|
||||||
if concurrency == 0 {
|
if concurrency == 0 {
|
||||||
concurrency = sourcefilesCount
|
concurrency = sourcefilesCount
|
||||||
}
|
}
|
||||||
|
counter := 0
|
||||||
index := 0
|
index := 0
|
||||||
|
errors := make(map[string]error)
|
||||||
channel := make(chan struct{}, concurrency)
|
channel := make(chan struct{}, concurrency)
|
||||||
for file, size := range sourcefiles {
|
for file, size := range sourcefiles {
|
||||||
channel <- struct{}{}
|
channel <- struct{}{}
|
||||||
|
@ -53,7 +55,12 @@ func Transfer() {
|
||||||
go func(file string, size int64, index int) {
|
go func(file string, size int64, index int) {
|
||||||
defer waitgroup.Done()
|
defer waitgroup.Done()
|
||||||
bar := createProgressBar(barcontainer, filepath.Base(file), size, int64(100), index, false)
|
bar := createProgressBar(barcontainer, filepath.Base(file), size, int64(100), index, false)
|
||||||
if transferFile(bar, file) {
|
error := transferFile(bar, file)
|
||||||
|
if error != nil {
|
||||||
|
errors[file] = error
|
||||||
|
bar.Abort(false)
|
||||||
|
bar.Wait()
|
||||||
|
} else {
|
||||||
counter++
|
counter++
|
||||||
}
|
}
|
||||||
totalbar.Increment()
|
totalbar.Increment()
|
||||||
|
@ -69,10 +76,16 @@ func Transfer() {
|
||||||
transferSpeedName += "/s"
|
transferSpeedName += "/s"
|
||||||
transferSize, transferSizeName := humanizeBytes(transferSize)
|
transferSize, transferSizeName := humanizeBytes(transferSize)
|
||||||
log.InfoTimed("transferred "+strconv.Itoa(counter)+" files, "+strconv.Itoa(int(transferSize))+" "+transferSizeName+" ("+strconv.FormatFloat(transferSpeed, 'f', 2, 64)+" "+transferSpeedName+")", timestamp.UnixMilli())
|
log.InfoTimed("transferred "+strconv.Itoa(counter)+" files, "+strconv.Itoa(int(transferSize))+" "+transferSizeName+" ("+strconv.FormatFloat(transferSpeed, 'f', 2, 64)+" "+transferSpeedName+")", timestamp.UnixMilli())
|
||||||
|
if len(errors) > 0 {
|
||||||
|
log.Info("encountered " + strconv.Itoa(len(errors)) + " errors")
|
||||||
|
for file, error := range errors {
|
||||||
|
log.Info(file, error.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// unexported function(s)
|
// unexported function(s)
|
||||||
func transferFile(bar *mpb.Bar, file string) bool {
|
func transferFile(bar *mpb.Bar, file string) error {
|
||||||
target := getTargetLocation(file)
|
target := getTargetLocation(file)
|
||||||
var arguments []string
|
var arguments []string
|
||||||
if len(settings.Password) > 0 {
|
if len(settings.Password) > 0 {
|
||||||
|
@ -89,16 +102,18 @@ func transferFile(bar *mpb.Bar, file string) bool {
|
||||||
target = settings.User + "@" + target
|
target = settings.User + "@" + target
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
arguments = append(arguments, "rsync", "-avz", "--mkpath", file, target, "--progress")
|
arguments = append(arguments, "rsync", "-avz", "--secluded-args", "--mkpath", file, target, "--progress")
|
||||||
cmd := exec.Command("sshpass", arguments...)
|
cmd := exec.Command("sshpass", arguments...)
|
||||||
stdout, stdoutError := cmd.StdoutPipe()
|
stdout, stdoutError := cmd.StdoutPipe()
|
||||||
stderr, stderrError := cmd.StderrPipe()
|
stderr, stderrError := cmd.StderrPipe()
|
||||||
cmd.Start()
|
cmd.Start()
|
||||||
if stdoutError != nil {
|
if stdoutError != nil {
|
||||||
log.Fatal(stdoutError.Error())
|
log.Warning("encountered an error opening remote stdout pipe", "file: "+file, stdoutError.Error())
|
||||||
|
return stderrError
|
||||||
}
|
}
|
||||||
if stderrError != nil {
|
if stderrError != nil {
|
||||||
log.Fatal(stderrError.Error())
|
log.Warning("encountered an error opening remote stderr pipe", "file: "+file, stdoutError.Error())
|
||||||
|
return stderrError
|
||||||
}
|
}
|
||||||
var resultBytes []byte
|
var resultBytes []byte
|
||||||
var waitgroup sync.WaitGroup
|
var waitgroup sync.WaitGroup
|
||||||
|
@ -110,7 +125,7 @@ func transferFile(bar *mpb.Bar, file string) bool {
|
||||||
readBytes, error := stdout.Read(tmp)
|
readBytes, error := stdout.Read(tmp)
|
||||||
if error != nil {
|
if error != nil {
|
||||||
if error != io.EOF {
|
if error != io.EOF {
|
||||||
log.Warning("encountered an error reading stdout", error.Error())
|
log.Warning("encountered an error reading stdout", "file: "+file, error.Error())
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
@ -126,7 +141,7 @@ func transferFile(bar *mpb.Bar, file string) bool {
|
||||||
tmp = strings.ReplaceAll(strings.TrimSpace(tmp), ".", "")
|
tmp = strings.ReplaceAll(strings.TrimSpace(tmp), ".", "")
|
||||||
bytes, error := strconv.ParseInt(tmp, 10, 64)
|
bytes, error := strconv.ParseInt(tmp, 10, 64)
|
||||||
if error != nil {
|
if error != nil {
|
||||||
log.Fatal("encountered an error converting the transferred bytes to int", error.Error())
|
log.Fatal("encountered an error converting the transferred bytes to int", "file: "+file, error.Error())
|
||||||
}
|
}
|
||||||
transferSize += bytes
|
transferSize += bytes
|
||||||
}
|
}
|
||||||
|
@ -156,17 +171,19 @@ func transferFile(bar *mpb.Bar, file string) bool {
|
||||||
}()
|
}()
|
||||||
errorBytes, stderrError := io.ReadAll(stderr)
|
errorBytes, stderrError := io.ReadAll(stderr)
|
||||||
if stderrError != nil {
|
if stderrError != nil {
|
||||||
log.Fatal(stderrError.Error())
|
log.Warning("encountered an error reading from remote stderr", "file: "+file, stdoutError.Error())
|
||||||
|
return stderrError
|
||||||
}
|
}
|
||||||
cmd.Wait()
|
cmd.Wait()
|
||||||
error := strings.Trim(string(errorBytes), "\n")
|
error := strings.Trim(string(errorBytes), "\n")
|
||||||
if len(error) > 0 {
|
if len(error) > 0 {
|
||||||
log.Fatal(error)
|
log.Warning("encountered an remote error", "file: "+file, error)
|
||||||
|
return errors.New(error)
|
||||||
}
|
}
|
||||||
waitgroup.Wait()
|
waitgroup.Wait()
|
||||||
stdout.Close()
|
stdout.Close()
|
||||||
stderr.Close()
|
stderr.Close()
|
||||||
return true
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getTargetLocation(sourceFile string) string {
|
func getTargetLocation(sourceFile string) string {
|
||||||
|
@ -182,16 +199,22 @@ func getTargetLocation(sourceFile string) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max int64, priority int, total bool) *mpb.Bar {
|
func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max int64, priority int, total bool) *mpb.Bar {
|
||||||
red, green, magenta, yellow := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow)
|
_, green, magenta, yellow, blue := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow), color.New(color.FgBlue)
|
||||||
barstyle := mpb.BarStyle().Lbound("").Filler("").Tip("").Padding(" ").Rbound("")
|
barstyle := mpb.BarStyle().Lbound("").Filler("").Tip("").Padding(" ").Rbound("")
|
||||||
defaultBarPrepend := mpb.PrependDecorators(
|
defaultBarPrepend := mpb.PrependDecorators(
|
||||||
decor.Name("[info] > "),
|
decor.Name("[info] > "),
|
||||||
decor.OnCompleteMeta(
|
decor.OnCompleteMeta(
|
||||||
decor.OnComplete(
|
decor.OnComplete(
|
||||||
decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(red)), ""+name,
|
decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(blue)), ""+name,
|
||||||
),
|
),
|
||||||
colorize(green),
|
colorize(green),
|
||||||
),
|
),
|
||||||
|
// decor.OnAbortMeta(
|
||||||
|
// decor.OnAbort(
|
||||||
|
// decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(blue)), ""+name,
|
||||||
|
// ),
|
||||||
|
// colorize(red),
|
||||||
|
// ),
|
||||||
)
|
)
|
||||||
defaultBarAppend := mpb.AppendDecorators(
|
defaultBarAppend := mpb.AppendDecorators(
|
||||||
decor.OnCompleteMeta(decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpaceR), colorize(yellow)),
|
decor.OnCompleteMeta(decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpaceR), colorize(yellow)),
|
||||||
|
@ -201,6 +224,12 @@ func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max
|
||||||
decor.OnComplete(
|
decor.OnComplete(
|
||||||
decor.Percentage(decor.WCSyncSpaceR), "",
|
decor.Percentage(decor.WCSyncSpaceR), "",
|
||||||
),
|
),
|
||||||
|
// decor.OnAbort(
|
||||||
|
// decor.Name("", decor.WCSyncSpaceR), "",
|
||||||
|
// ),
|
||||||
|
// decor.OnAbort(
|
||||||
|
// decor.Percentage(decor.WCSyncSpace), "",
|
||||||
|
// ),
|
||||||
)
|
)
|
||||||
totalBarPrepend := mpb.PrependDecorators(
|
totalBarPrepend := mpb.PrependDecorators(
|
||||||
decor.Name("[info] > "),
|
decor.Name("[info] > "),
|
||||||
|
|
Loading…
Reference in a new issue