fixed rsync command for files with special character (fu***ing backticks)

.vscode/launch.json

@@ -8,12 +8,33 @@
             "mode": "auto",
             "program": "${workspaceFolder}/main.go",
             "args": [
-                "/home/velvettear/downloads/music",
+                "--verbose",
-                "192.168.1.11:/tmp",
                 "--password",
                 "$Velvet90",
                 "--concurrency",
-                "4",
+                "24",
+                "--delay",
+                "100",
+                "/mnt/kingston/downloads/music/*",
+                "192.168.1.11:/share/tmp/music",
+            ],
+            "console": "integratedTerminal"
+        },
+        {
+            "name": "gosync-remote",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/main.go",
+            "args": [
+                "192.168.1.11:/share/music/Fu Manchu/*",
+                "/tmp",
+                "--password",
+                "$Velvet90",
+                "--concurrency",
+                "12",
+                "--delay",
+                "250",
                 "--verbose",
             ],
             "console": "integratedTerminal"

README.md

@@ -14,9 +14,16 @@ a simple wrapper for concurrent rsync processes written in golang
 
 ### options
 
-| short | long          | description                                 |
+| short | long          | description                                     | default             |
-| ----- | ------------- | ------------------------------------------- |
+| ----- | ------------- | ----------------------------------------------- | ------------------- |
-| -u    | --user        | set user for ssh / rsync                    |
+| -u    | --user        | set user for ssh / rsync                        |                     |
-| -p    | --password    | set password for ssh / rsync                |
+| -p    | --password    | set password for ssh / rsync                    |                     |
-| -c    | --concurrency | set limit for concurrent rsync processes    |
+| -c    | --concurrency | set limit for concurrent rsync processes        | number of cpu cores |
-| -v    | --verbose     | enable verbose / debug output               |
+| -d    | --delay       | set the delay between rsync connections (in ms) | 100                 |
+| -v    | --verbose     | enable verbose / debug output                   |                     |
+
+### troubleshooting
+
+**make sure to wrap your `[source]` and `[target]` in `"` to avoid problems with paths and / or globbing.**
+
+if you experience errors like `kex_exchange_identification: read: Connection reset by peer` it may be helpful to increase the default delay (100ms) between rsync connections or decrease the concurrency.

log/log.go

@@ -14,7 +14,7 @@ const LEVEL_WARNING = 2
 const LEVEL_ERROR = 3
 const LEVEL_FATAL = 4
 
-var logLevel = 0
+var logLevel = 1
 
 // exported functions
 func SetLogLevel(level int) {
@@ -73,7 +73,6 @@ func trace(level int, timestamp int64, message string, extras ...string) {
 		message += " (" + suffix + ")"
 	}
 	if timestamp >= 0 {
-
 		message += " [" + strconv.Itoa(int(time.Now().UnixMilli()-timestamp)) + "ms" + "]"
 	}
 	fmt.Println(buildLogMessage(getPrefixForLogLevel(level), message))

main.go

@@ -8,6 +8,8 @@ import (
 	"velvettear/gosync/tools"
 )
 
+// version := "0.1"
+
 func main() {
 	timestamp := time.Now()
 	settings.Initialize()

settings/arguments.go

@@ -5,82 +5,93 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 	"velvettear/gosync/help"
 	"velvettear/gosync/log"
 )
 
 // exported function(s)
 func Initialize() {
-	os.Args = os.Args[1:]
 	var arguments []string
-	for index, arg := range os.Args {
+	for index := 1; index < len(os.Args); index++ {
-		switch strings.ToLower(arg) {
+		arg := strings.ToLower(os.Args[index])
+		if arg != "-v" && arg != "--verbose" {
+			arguments = append(arguments, os.Args[index])
+			continue
+		}
+		setVerbose(true)
+	}
+	var filteredArguments []string
+	for index := 0; index < len(arguments); index++ {
+		switch strings.ToLower(arguments[index]) {
 		case "-h":
 			fallthrough
 		case "--help":
 			help.Print()
 			os.Exit(0)
-		case "-v":
-			fallthrough
-		case "--verbose":
-			setVerbose(true)
 		case "-c":
 			fallthrough
 		case "--concurrency":
-			var concurrency int
+			index++
-			tmpIndex := index + 1
+			if index < len(arguments) {
-			if tmpIndex < len(os.Args) {
+				concurrency, error := strconv.Atoi(arguments[index])
-				tmp, error := strconv.Atoi(os.Args[tmpIndex])
+				if error != nil {
-				if error == nil {
+					break
-					concurrency = tmp
-				}
-			}
-			if concurrency == 0 {
-				concurrency = runtime.NumCPU()
 				}
 				setConcurrency(concurrency)
+			}
+		case "-d":
+			fallthrough
+		case "--delay":
+			index++
+			if index < len(arguments) {
+				delay, error := strconv.Atoi(arguments[index])
+				if error != nil {
+					break
+				}
+				setDelay(time.Duration(delay) * time.Millisecond)
+			}
 		case "-p":
 			fallthrough
 		case "--password":
-			tmpIndex := index + 1
+			index++
-			if index > len(os.Args) {
+			if index > len(arguments) {
 				break
 			}
-			setPassword(os.Args[tmpIndex])
+			setPassword(arguments[index])
 		case "-u":
 			fallthrough
 		case "--user":
-			tmpIndex := index + 1
+			index++
-			if index > len(os.Args) {
+			if index > len(arguments) {
 				break
 			}
-			setUser(os.Args[tmpIndex])
+			setUser(arguments[index])
 		default:
-			arguments = append(arguments, arg)
+			filteredArguments = append(filteredArguments, arguments[index])
 		}
 	}
-	if len(os.Args) < 2 {
+	if len(arguments) < 2 {
 		log.Fatal("error: missing arguments")
 	}
-	setSource(arguments[0])
+	setSource(filteredArguments[0])
-	setTarget(arguments[1])
+	setTarget(filteredArguments[1])
-	_, error := os.Stat(Source)
+	if !SourceIsRemote() {
+		source, _ := strings.CutSuffix(Source, "/*")
+		_, error := os.Stat(source)
 		if os.IsNotExist(error) {
-		log.Fatal("given source does not exist", Source)
+			log.Fatal("given source does not exist", source)
 		}
-	if !Verbose {
-		setVerbose(false)
 	}
+	setDefaults()
 }
 
 // unexported function(s)
-func removeArgument(index int) {
+func setDefaults() {
-	removeArguments(index, 0, 0)
+	if Concurrency == 0 {
-}
+		setConcurrency(runtime.NumCPU())
-
+	}
-func removeArguments(index int, before int, after int) {
+	if Delay == 0 {
-	// derp := index - 1 - before
+		setDelay(100)
-	copyArgs := os.Args[0 : index-before]
+	}
-	copyArgs = append(copyArgs, os.Args[index+1+after:]...)
-	os.Args = copyArgs
 }

settings/variables.go

@@ -3,6 +3,7 @@ package settings
 import (
 	"strconv"
 	"strings"
+	"time"
 	"velvettear/gosync/log"
 )
 
@@ -11,15 +12,28 @@ var Verbose bool
 var Source string
 var Target string
 var Concurrency int
+var Delay time.Duration
 var Password string
 var User string
 
 // exported function(s)
 func TargetIsRemote() bool {
-	return strings.Contains(Target, ":")
+	return isRemote(Target)
+}
+
+func SourceIsRemote() bool {
+	return isRemote(Source)
+}
+
+func SourceIsWildcard() bool {
+	return strings.HasSuffix(Source, "/*")
 }
 
 // unexported function(s)
+func isRemote(target string) bool {
+	return strings.Contains(target, ":")
+}
+
 func setVerbose(verbose bool) {
 	Verbose = verbose
 	if Verbose {
@@ -45,6 +59,13 @@ func setConcurrency(concurrency int) {
 	log.Debug("set concurrency", strconv.Itoa(Concurrency))
 }
 
+func setDelay(delay time.Duration) {
+	Delay = delay
+	ms := Delay.Milliseconds()
+	derp := strconv.FormatInt(ms, 10)
+	log.Debug("set delay", derp)
+}
+
 func setPassword(password string) {
 	Password = password
 	log.Debug("set password", Password)

tools/rsync.go

@@ -1,6 +1,7 @@
 package tools
 
 import (
+	"errors"
 	"io"
 	"os"
 	"os/exec"
@@ -17,83 +18,102 @@ import (
 	"github.com/vbauerster/mpb/v8/decor"
 )
 
-var transferSize float64
+var transferSize int64
+
+const byteToMegabyte = 1048576
+const megabyteToGigabyte = 1024
 
 // exported function(s)
 func Transfer() {
-	transferSize = 0
 	sourcefiles := getSourceFiles()
 	sourcefilesCount := len(sourcefiles)
+	if sourcefilesCount <= 0 {
+		log.Info("nothing to do - exiting...")
+		os.Exit(0)
+	}
+	sourceFileSize, sourceFileSizeName := humanizeBytes(getSourceFileSize())
 	var waitgroup sync.WaitGroup
 	waitgroup.Add(sourcefilesCount)
 	barcontainer := mpb.New(
 		mpb.WithWaitGroup(&waitgroup),
 	)
 	timestamp := time.Now()
-	counter := 0
+	totalbar := createProgressBar(barcontainer, strconv.FormatFloat(sourceFileSize, 'f', 2, 64)+sourceFileSizeName+" 󰇙 Total", int64(0), int64(sourcefilesCount), sourcefilesCount+1, true)
-	totalbar := createProgressBar(barcontainer, "Total", int64(0), int64(sourcefilesCount), sourcefilesCount+1, true)
 	concurrency := settings.Concurrency
 	if concurrency == 0 {
 		concurrency = sourcefilesCount
 	}
+	counter := 0
+	index := 0
+	errors := make(map[string]error)
 	channel := make(chan struct{}, concurrency)
-	for index, file := range sourcefiles {
+	for file, size := range sourcefiles {
 		channel <- struct{}{}
 		if index > 0 {
-			time.Sleep(100 * time.Millisecond)
+			time.Sleep(settings.Delay)
 		}
-		go func(index int, file string) {
+		go func(file string, size int64, index int) {
 			defer waitgroup.Done()
-			stats, error := os.Stat(file)
+			bar := createProgressBar(barcontainer, filepath.Base(file), size, int64(100), index, false)
+			error := transferFile(bar, file)
 			if error != nil {
-				log.Warning("encountered an error getting file size", error.Error())
+				errors[file] = error
-				return
+				bar.Abort(false)
-			}
+				bar.Wait()
-			bar := createProgressBar(barcontainer, filepath.Base(file), stats.Size(), int64(100), index, false)
+			} else {
-			if transferFile(bar, file) {
 				counter++
 			}
 			totalbar.Increment()
 			<-channel
-		}(index, file)
+		}(file, size, index)
+		index++
 	}
 	barcontainer.Wait()
 	timeDifference := time.Since(timestamp)
-	transferSpeedName := "bytes/sec"
+	transferSpeedName := "B/sec"
-	transferSpeed := transferSize / timeDifference.Seconds()
+	transferSpeed := float64(transferSize) / timeDifference.Seconds()
-	if transferSpeed > 1048576 {
+	transferSpeed, transferSpeedName = humanizeBytes(int64(transferSpeed))
-		transferSpeed = transferSpeed / 1048576
+	transferSpeedName += "/s"
-		transferSpeedName = "mb/s"
+	transferSize, transferSizeName := humanizeBytes(transferSize)
-	}
-	transferSizeName := "bytes"
-	if transferSize > 1048576 {
-		transferSize = transferSize / 1048576
-		transferSizeName = "mb"
-	}
 	log.InfoTimed("transferred "+strconv.Itoa(counter)+" files, "+strconv.Itoa(int(transferSize))+" "+transferSizeName+" ("+strconv.FormatFloat(transferSpeed, 'f', 2, 64)+" "+transferSpeedName+")", timestamp.UnixMilli())
+	if len(errors) > 0 {
+		log.Info("encountered " + strconv.Itoa(len(errors)) + " errors")
+		for file, error := range errors {
+			log.Info(file, error.Error())
+		}
+	}
 }
 
 // unexported function(s)
-func transferFile(bar *mpb.Bar, file string) bool {
+func transferFile(bar *mpb.Bar, file string) error {
 	target := getTargetLocation(file)
 	var arguments []string
 	if len(settings.Password) > 0 {
 		arguments = append(arguments, "-p", settings.Password)
 	}
-	arguments = append(arguments, "rsync", "-avz", "--mkpath", file)
+	if settings.SourceIsRemote() {
+		remote, _, _ := strings.Cut(settings.Source, ":")
+		file = remote + ":" + file
+		if len(settings.User) > 0 {
+			file = settings.User + "@" + file
+		}
+	} else {
 		if len(settings.User) > 0 {
 			target = settings.User + "@" + target
 		}
-	arguments = append(arguments, target, "--progress")
+	}
+	arguments = append(arguments, "rsync", "-avz", "--secluded-args", "--mkpath", file, target, "--progress")
 	cmd := exec.Command("sshpass", arguments...)
 	stdout, stdoutError := cmd.StdoutPipe()
 	stderr, stderrError := cmd.StderrPipe()
 	cmd.Start()
 	if stdoutError != nil {
-		log.Fatal(stdoutError.Error())
+		log.Warning("encountered an error opening remote stdout pipe", "file: "+file, stdoutError.Error())
+		return stderrError
 	}
 	if stderrError != nil {
-		log.Fatal(stderrError.Error())
+		log.Warning("encountered an error opening remote stderr pipe", "file: "+file, stdoutError.Error())
+		return stderrError
 	}
 	var resultBytes []byte
 	var waitgroup sync.WaitGroup
@@ -105,7 +125,7 @@ func transferFile(bar *mpb.Bar, file string) bool {
 			readBytes, error := stdout.Read(tmp)
 			if error != nil {
 				if error != io.EOF {
-					log.Warning("encountered an error reading stdout", error.Error())
+					log.Warning("encountered an error reading stdout", "file: "+file, error.Error())
 				}
 				break
 			}
@@ -119,9 +139,9 @@ func transferFile(bar *mpb.Bar, file string) bool {
 				_, tmp, _ := strings.Cut(lowerline, "sent")
 				tmp, _, _ = strings.Cut(tmp, "bytes")
 				tmp = strings.ReplaceAll(strings.TrimSpace(tmp), ".", "")
-				bytes, error := strconv.ParseFloat(tmp, 64)
+				bytes, error := strconv.ParseInt(tmp, 10, 64)
 				if error != nil {
-					log.Fatal("encountered an error converting the transferred bytes to int", error.Error())
+					log.Fatal("encountered an error converting the transferred bytes to int", "file: "+file, error.Error())
 				}
 				transferSize += bytes
 			}
@@ -151,46 +171,65 @@ func transferFile(bar *mpb.Bar, file string) bool {
 	}()
 	errorBytes, stderrError := io.ReadAll(stderr)
 	if stderrError != nil {
-		log.Fatal(stderrError.Error())
+		log.Warning("encountered an error reading from remote stderr", "file: "+file, stdoutError.Error())
+		return stderrError
 	}
 	cmd.Wait()
 	error := strings.Trim(string(errorBytes), "\n")
 	if len(error) > 0 {
-		log.Fatal(error)
+		log.Warning("encountered an remote error", "file: "+file, error)
+		return errors.New(error)
 	}
 	waitgroup.Wait()
 	stdout.Close()
 	stderr.Close()
-	return true
+	return nil
 }
 
-func getTargetLocation(source string) string {
+func getTargetLocation(sourceFile string) string {
-	if source == settings.Source {
+	if sourceFile == settings.Source {
-		return filepath.Join(settings.Target, filepath.Base(source))
+		return filepath.Join(settings.Target, filepath.Base(sourceFile))
 	}
-	return filepath.Join(settings.Target, strings.Replace(source, filepath.Dir(settings.Source), "", 1))
+	source, _ := strings.CutSuffix(settings.Source, "/*")
+	if settings.SourceIsRemote() {
+		_, source, _ = strings.Cut(source, ":")
+		source = filepath.Dir(source)
+	}
+	return filepath.Join(settings.Target, strings.Replace(sourceFile, source, "", 1))
 }
 
 func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max int64, priority int, total bool) *mpb.Bar {
-	red, green, magenta, yellow := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow)
+	_, green, magenta, yellow, blue := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow), color.New(color.FgBlue)
 	barstyle := mpb.BarStyle().Lbound("").Filler("󰝤").Tip("").Padding(" ").Rbound("")
 	defaultBarPrepend := mpb.PrependDecorators(
 		decor.Name("[info] > "),
 		decor.OnCompleteMeta(
 			decor.OnComplete(
-				decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(red)), ""+name,
+				decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(blue)), ""+name,
 			),
 			colorize(green),
 		),
+		// decor.OnAbortMeta(
+		// 	decor.OnAbort(
+		// 		decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(blue)), ""+name,
+		// 	),
+		// 	colorize(red),
+		// ),
 	)
 	defaultBarAppend := mpb.AppendDecorators(
 		decor.OnCompleteMeta(decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpaceR), colorize(yellow)),
 		decor.OnComplete(
-			decor.Name("|", decor.WCSyncSpaceR), "",
+			decor.Name("󰇙", decor.WCSyncSpaceR), "",
 		),
 		decor.OnComplete(
 			decor.Percentage(decor.WCSyncSpaceR), "",
 		),
+		// decor.OnAbort(
+		// 	decor.Name("󰇙", decor.WCSyncSpaceR), "",
+		// ),
+		// decor.OnAbort(
+		// 	decor.Percentage(decor.WCSyncSpace), "",
+		// ),
 	)
 	totalBarPrepend := mpb.PrependDecorators(
 		decor.Name("[info] > "),
@@ -229,8 +268,16 @@ func colorize(c *color.Color) func(string) string {
 	}
 }
 
-func decolorize() func(string) string {
+func humanizeBytes(bytes int64) (float64, string) {
-	return func(s string) string {
+	name := "B"
-		return ""
+	size := float64(bytes)
+	if size >= byteToMegabyte {
+		name = "MB"
+		size = size / byteToMegabyte
 	}
+	if size >= megabyteToGigabyte {
+		name = "GB"
+		size = size / megabyteToGigabyte
+	}
+	return size, name
 }

tools/scanner.go

@@ -1,34 +1,97 @@
 package tools
 
 import (
+	"errors"
+	"io"
 	"io/fs"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 	"velvettear/gosync/log"
 	"velvettear/gosync/settings"
 )
 
-var sourceFiles []string
+var sourceFiles map[string]int64
 
 // unexported function(s)
-func getSourceFiles() []string {
+func getSourceFiles() map[string]int64 {
 	timestamp := time.Now()
-	stats, error := os.Stat(settings.Source)
+	sourceFiles = map[string]int64{}
+	if settings.SourceIsRemote() {
+		log.Info("getting the list of remote source files...")
+		error := fillRemoteSourceFiles()
 		if error != nil {
-		log.Error("encountered an error getting the stats for the source", error.Error())
+			log.Fatal("encountered an error getting the list of remote source files", error.Error())
+		}
+	} else {
+		source, _ := strings.CutSuffix(settings.Source, "/*")
+		stats, error := os.Stat(source)
+		if error != nil {
+			log.Fatal("encountered an error getting stats for source", error.Error())
 		}
 		if stats.IsDir() {
-		log.Info("scanning source...", settings.Source)
+			log.Info("scanning source...", source)
-		filepath.WalkDir(settings.Source, fillSourceFiles)
+			filepath.WalkDir(source, fillSourceFiles)
-		log.InfoTimed("found "+strconv.Itoa(len(sourceFiles))+" source files", timestamp.UnixMilli())
 		} else {
-		sourceFiles = append(sourceFiles, settings.Source)
+			sourceFiles[settings.Source] = stats.Size()
 		}
+	}
+	log.InfoTimed("found "+strconv.Itoa(len(sourceFiles))+" source files", timestamp.UnixMilli())
 	return sourceFiles
 }
 
+func fillRemoteSourceFiles() error {
+	var arguments []string
+	if len(settings.Password) > 0 {
+		arguments = append(arguments, "-p", settings.Password)
+	}
+	arguments = append(arguments, "ssh")
+	remote, path, _ := strings.Cut(settings.Source, ":")
+	path, _ = strings.CutSuffix(path, "/*")
+	if len(settings.User) > 0 {
+		remote = settings.User + "@" + remote
+	}
+	arguments = append(arguments, remote, "find", "\""+path+"\"", "-type", "f", "-exec", "du", "-b", "{}", "\\;")
+	cmd := exec.Command("sshpass", arguments...)
+	stdout, stdoutError := cmd.StdoutPipe()
+	stderr, stderrError := cmd.StderrPipe()
+	cmd.Start()
+	if stdoutError != nil {
+		return stdoutError
+	}
+	if stderrError != nil {
+		return stderrError
+	}
+	outBytes, stdoutError := io.ReadAll(stdout)
+	if stdoutError != nil {
+		return stdoutError
+	}
+	errorBytes, stderrError := io.ReadAll(stderr)
+	if stderrError != nil {
+		return stderrError
+	}
+	cmd.Wait()
+	error := strings.TrimSpace(string(errorBytes))
+	if len(error) > 0 {
+		return errors.New(error)
+	}
+	for _, line := range strings.Split(string(outBytes), "\n") {
+		if !strings.Contains(line, "\t") {
+			continue
+		}
+		parts := strings.Split(line, "\t")
+		size, error := strconv.ParseInt(parts[0], 10, 64)
+		if error != nil {
+			log.Warning("encountered an error getting the file size for file '"+path+"'", error.Error())
+		}
+		sourceFiles[parts[1]] = size
+	}
+	return nil
+}
+
 func fillSourceFiles(path string, dir fs.DirEntry, err error) error {
 	if err != nil {
 		return err
@@ -36,6 +99,19 @@ func fillSourceFiles(path string, dir fs.DirEntry, err error) error {
 	if dir.IsDir() {
 		return nil
 	}
-	sourceFiles = append(sourceFiles, path)
+	stats, error := os.Stat(path)
+	if error != nil {
+		log.Fatal("encountered an error getting stats for file", error.Error())
+		return nil
+	}
+	sourceFiles[path] = stats.Size()
 	return nil
 }
+
+func getSourceFileSize() int64 {
+	var total int64
+	for _, size := range sourceFiles {
+		total += size
+	}
+	return total
+}

tools/ssh.go

@@ -11,7 +11,12 @@ import (
 
 // exported function(s)
 func TestConnection() error {
-	if !settings.TargetIsRemote() {
+	var remote string
+	if settings.SourceIsRemote() {
+		remote, _, _ = strings.Cut(settings.Source, ":")
+	} else if settings.TargetIsRemote() {
+		remote, _, _ = strings.Cut(settings.Target, ":")
+	} else {
 		return nil
 	}
 	var arguments []string
@@ -19,11 +24,10 @@ func TestConnection() error {
 		arguments = append(arguments, "-p", settings.Password)
 	}
 	arguments = append(arguments, "ssh")
-	target, _, _ := strings.Cut(settings.Target, ":")
 	if len(settings.User) > 0 {
-		target = settings.User + "@" + target
+		remote = settings.User + "@" + remote
 	}
-	arguments = append(arguments, target, "'exit'")
+	arguments = append(arguments, remote, "exit")
 	cmd := exec.Command("sshpass", arguments...)
 	stdout, stdoutError := cmd.StdoutPipe()
 	stderr, stderrError := cmd.StderrPipe()