.vscode/launch.json

@@ -8,33 +8,12 @@
             "mode": "auto",
             "program": "${workspaceFolder}/main.go",
             "args": [
-                "--verbose",
+                "/home/velvettear/downloads/music",
+                "192.168.1.11:/tmp",
                 "--password",
                 "$Velvet90",
                 "--concurrency",
-                "24",
-                "--delay",
-                "100",
-                "/mnt/kingston/downloads/music/*",
-                "192.168.1.11:/share/tmp/music",
-            ],
-            "console": "integratedTerminal"
-        },
-        {
-            "name": "gosync-remote",
-            "type": "go",
-            "request": "launch",
-            "mode": "auto",
-            "program": "${workspaceFolder}/main.go",
-            "args": [
-                "192.168.1.11:/share/music/Fu Manchu/*",
-                "/tmp",
-                "--password",
-                "$Velvet90",
-                "--concurrency",
-                "12",
-                "--delay",
-                "250",
+                "4",
                 "--verbose",
             ],
             "console": "integratedTerminal"

README.md

@@ -14,16 +14,9 @@ a simple wrapper for concurrent rsync processes written in golang
 
 ### options
 
-| short | long          | description                                     | default             |
-| ----- | ------------- | ----------------------------------------------- | ------------------- |
-| -u    | --user        | set user for ssh / rsync                        |                     |
-| -p    | --password    | set password for ssh / rsync                    |                     |
-| -c    | --concurrency | set limit for concurrent rsync processes        | number of cpu cores |
-| -d    | --delay       | set the delay between rsync connections (in ms) | 100                 |
-| -v    | --verbose     | enable verbose / debug output                   |                     |
-
-### troubleshooting
-
-**make sure to wrap your `[source]` and `[target]` in `"` to avoid problems with paths and / or globbing.**
-
-if you experience errors like `kex_exchange_identification: read: Connection reset by peer` it may be helpful to increase the default delay (100ms) between rsync connections or decrease the concurrency.
+| short | long          | description                                 |
+| ----- | ------------- | ------------------------------------------- |
+| -u    | --user        | set user for ssh / rsync                    |
+| -p    | --password    | set password for ssh / rsync                |
+| -c    | --concurrency | set limit for concurrent rsync processes    |
+| -v    | --verbose     | enable verbose / debug output               |

log/log.go

@@ -14,7 +14,7 @@ const LEVEL_WARNING = 2
 const LEVEL_ERROR = 3
 const LEVEL_FATAL = 4
 
-var logLevel = 1
+var logLevel = 0
 
 // exported functions
 func SetLogLevel(level int) {
@@ -73,6 +73,7 @@ func trace(level int, timestamp int64, message string, extras ...string) {
 		message += " (" + suffix + ")"
 	}
 	if timestamp >= 0 {
+
 		message += " [" + strconv.Itoa(int(time.Now().UnixMilli()-timestamp)) + "ms" + "]"
 	}
 	fmt.Println(buildLogMessage(getPrefixForLogLevel(level), message))

main.go

@@ -8,8 +8,6 @@ import (
 	"velvettear/gosync/tools"
 )
 
-// version := "0.1"
-
 func main() {
 	timestamp := time.Now()
 	settings.Initialize()

settings/arguments.go

@@ -5,93 +5,82 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"time"
 	"velvettear/gosync/help"
 	"velvettear/gosync/log"
 )
 
 // exported function(s)
 func Initialize() {
+	os.Args = os.Args[1:]
 	var arguments []string
-	for index := 1; index < len(os.Args); index++ {
-		arg := strings.ToLower(os.Args[index])
-		if arg != "-v" && arg != "--verbose" {
-			arguments = append(arguments, os.Args[index])
-			continue
-		}
-		setVerbose(true)
-	}
-	var filteredArguments []string
-	for index := 0; index < len(arguments); index++ {
-		switch strings.ToLower(arguments[index]) {
+	for index, arg := range os.Args {
+		switch strings.ToLower(arg) {
 		case "-h":
 			fallthrough
 		case "--help":
 			help.Print()
 			os.Exit(0)
+		case "-v":
+			fallthrough
+		case "--verbose":
+			setVerbose(true)
 		case "-c":
 			fallthrough
 		case "--concurrency":
-			index++
-			if index < len(arguments) {
-				concurrency, error := strconv.Atoi(arguments[index])
-				if error != nil {
-					break
+			var concurrency int
+			tmpIndex := index + 1
+			if tmpIndex < len(os.Args) {
+				tmp, error := strconv.Atoi(os.Args[tmpIndex])
+				if error == nil {
+					concurrency = tmp
 				}
-				setConcurrency(concurrency)
 			}
-		case "-d":
-			fallthrough
-		case "--delay":
-			index++
-			if index < len(arguments) {
-				delay, error := strconv.Atoi(arguments[index])
-				if error != nil {
-					break
-				}
-				setDelay(time.Duration(delay) * time.Millisecond)
+			if concurrency == 0 {
+				concurrency = runtime.NumCPU()
 			}
+			setConcurrency(concurrency)
 		case "-p":
 			fallthrough
 		case "--password":
-			index++
-			if index > len(arguments) {
+			tmpIndex := index + 1
+			if index > len(os.Args) {
 				break
 			}
-			setPassword(arguments[index])
+			setPassword(os.Args[tmpIndex])
 		case "-u":
 			fallthrough
 		case "--user":
-			index++
-			if index > len(arguments) {
+			tmpIndex := index + 1
+			if index > len(os.Args) {
 				break
 			}
-			setUser(arguments[index])
+			setUser(os.Args[tmpIndex])
 		default:
-			filteredArguments = append(filteredArguments, arguments[index])
+			arguments = append(arguments, arg)
 		}
 	}
-	if len(arguments) < 2 {
+	if len(os.Args) < 2 {
 		log.Fatal("error: missing arguments")
 	}
-	setSource(filteredArguments[0])
-	setTarget(filteredArguments[1])
-	if !SourceIsRemote() {
-		source, _ := strings.CutSuffix(Source, "/*")
-		_, error := os.Stat(source)
-		if os.IsNotExist(error) {
-			log.Fatal("given source does not exist", source)
-		}
+	setSource(arguments[0])
+	setTarget(arguments[1])
+	_, error := os.Stat(Source)
+	if os.IsNotExist(error) {
+		log.Fatal("given source does not exist", Source)
+	}
+	if !Verbose {
+		setVerbose(false)
 	}
-	setDefaults()
 }
 
 // unexported function(s)
-func setDefaults() {
-	if Concurrency == 0 {
-		setConcurrency(runtime.NumCPU())
-	}
-	if Delay == 0 {
-		setDelay(100)
-	}
+func removeArgument(index int) {
+	removeArguments(index, 0, 0)
+}
+
+func removeArguments(index int, before int, after int) {
+	// derp := index - 1 - before
+	copyArgs := os.Args[0 : index-before]
+	copyArgs = append(copyArgs, os.Args[index+1+after:]...)
+	os.Args = copyArgs
 }

settings/variables.go

@@ -3,7 +3,6 @@ package settings
 import (
 	"strconv"
 	"strings"
-	"time"
 	"velvettear/gosync/log"
 )
 
@@ -12,28 +11,15 @@ var Verbose bool
 var Source string
 var Target string
 var Concurrency int
-var Delay time.Duration
 var Password string
 var User string
 
 // exported function(s)
 func TargetIsRemote() bool {
-	return isRemote(Target)
-}
-
-func SourceIsRemote() bool {
-	return isRemote(Source)
-}
-
-func SourceIsWildcard() bool {
-	return strings.HasSuffix(Source, "/*")
+	return strings.Contains(Target, ":")
 }
 
 // unexported function(s)
-func isRemote(target string) bool {
-	return strings.Contains(target, ":")
-}
-
 func setVerbose(verbose bool) {
 	Verbose = verbose
 	if Verbose {
@@ -59,13 +45,6 @@ func setConcurrency(concurrency int) {
 	log.Debug("set concurrency", strconv.Itoa(Concurrency))
 }
 
-func setDelay(delay time.Duration) {
-	Delay = delay
-	ms := Delay.Milliseconds()
-	derp := strconv.FormatInt(ms, 10)
-	log.Debug("set delay", derp)
-}
-
 func setPassword(password string) {
 	Password = password
 	log.Debug("set password", Password)

tools/rsync.go

@@ -1,7 +1,6 @@
 package tools
 
 import (
-	"errors"
 	"io"
 	"os"
 	"os/exec"
@@ -18,102 +17,83 @@ import (
 	"github.com/vbauerster/mpb/v8/decor"
 )
 
-var transferSize int64
-
-const byteToMegabyte = 1048576
-const megabyteToGigabyte = 1024
+var transferSize float64
 
 // exported function(s)
 func Transfer() {
+	transferSize = 0
 	sourcefiles := getSourceFiles()
 	sourcefilesCount := len(sourcefiles)
-	if sourcefilesCount <= 0 {
-		log.Info("nothing to do - exiting...")
-		os.Exit(0)
-	}
-	sourceFileSize, sourceFileSizeName := humanizeBytes(getSourceFileSize())
 	var waitgroup sync.WaitGroup
 	waitgroup.Add(sourcefilesCount)
 	barcontainer := mpb.New(
 		mpb.WithWaitGroup(&waitgroup),
 	)
 	timestamp := time.Now()
-	totalbar := createProgressBar(barcontainer, strconv.FormatFloat(sourceFileSize, 'f', 2, 64)+sourceFileSizeName+" 󰇙 Total", int64(0), int64(sourcefilesCount), sourcefilesCount+1, true)
+	counter := 0
+	totalbar := createProgressBar(barcontainer, "Total", int64(0), int64(sourcefilesCount), sourcefilesCount+1, true)
 	concurrency := settings.Concurrency
 	if concurrency == 0 {
 		concurrency = sourcefilesCount
 	}
-	counter := 0
-	index := 0
-	errors := make(map[string]error)
 	channel := make(chan struct{}, concurrency)
-	for file, size := range sourcefiles {
+	for index, file := range sourcefiles {
 		channel <- struct{}{}
 		if index > 0 {
-			time.Sleep(settings.Delay)
+			time.Sleep(100 * time.Millisecond)
 		}
-		go func(file string, size int64, index int) {
+		go func(index int, file string) {
 			defer waitgroup.Done()
-			bar := createProgressBar(barcontainer, filepath.Base(file), size, int64(100), index, false)
-			error := transferFile(bar, file)
+			stats, error := os.Stat(file)
 			if error != nil {
-				errors[file] = error
-				bar.Abort(false)
-				bar.Wait()
-			} else {
+				log.Warning("encountered an error getting file size", error.Error())
+				return
+			}
+			bar := createProgressBar(barcontainer, filepath.Base(file), stats.Size(), int64(100), index, false)
+			if transferFile(bar, file) {
 				counter++
 			}
 			totalbar.Increment()
 			<-channel
-		}(file, size, index)
-		index++
+		}(index, file)
 	}
 	barcontainer.Wait()
 	timeDifference := time.Since(timestamp)
-	transferSpeedName := "B/sec"
-	transferSpeed := float64(transferSize) / timeDifference.Seconds()
-	transferSpeed, transferSpeedName = humanizeBytes(int64(transferSpeed))
-	transferSpeedName += "/s"
-	transferSize, transferSizeName := humanizeBytes(transferSize)
-	log.InfoTimed("transferred "+strconv.Itoa(counter)+" files, "+strconv.Itoa(int(transferSize))+" "+transferSizeName+" ("+strconv.FormatFloat(transferSpeed, 'f', 2, 64)+" "+transferSpeedName+")", timestamp.UnixMilli())
-	if len(errors) > 0 {
-		log.Info("encountered " + strconv.Itoa(len(errors)) + " errors")
-		for file, error := range errors {
-			log.Info(file, error.Error())
-		}
+	transferSpeedName := "bytes/sec"
+	transferSpeed := transferSize / timeDifference.Seconds()
+	if transferSpeed > 1048576 {
+		transferSpeed = transferSpeed / 1048576
+		transferSpeedName = "mb/s"
 	}
+	transferSizeName := "bytes"
+	if transferSize > 1048576 {
+		transferSize = transferSize / 1048576
+		transferSizeName = "mb"
+	}
+	log.InfoTimed("transferred "+strconv.Itoa(counter)+" files, "+strconv.Itoa(int(transferSize))+" "+transferSizeName+" ("+strconv.FormatFloat(transferSpeed, 'f', 2, 64)+" "+transferSpeedName+")", timestamp.UnixMilli())
 }
 
 // unexported function(s)
-func transferFile(bar *mpb.Bar, file string) error {
+func transferFile(bar *mpb.Bar, file string) bool {
 	target := getTargetLocation(file)
 	var arguments []string
 	if len(settings.Password) > 0 {
 		arguments = append(arguments, "-p", settings.Password)
 	}
-	if settings.SourceIsRemote() {
-		remote, _, _ := strings.Cut(settings.Source, ":")
-		file = remote + ":" + file
-		if len(settings.User) > 0 {
-			file = settings.User + "@" + file
-		}
-	} else {
-		if len(settings.User) > 0 {
-			target = settings.User + "@" + target
-		}
+	arguments = append(arguments, "rsync", "-avz", "--mkpath", file)
+	if len(settings.User) > 0 {
+		target = settings.User + "@" + target
 	}
-	arguments = append(arguments, "rsync", "-avz", "--secluded-args", "--mkpath", file, target, "--progress")
+	arguments = append(arguments, target, "--progress")
 	cmd := exec.Command("sshpass", arguments...)
 	stdout, stdoutError := cmd.StdoutPipe()
 	stderr, stderrError := cmd.StderrPipe()
 	cmd.Start()
 	if stdoutError != nil {
-		log.Warning("encountered an error opening remote stdout pipe", "file: "+file, stdoutError.Error())
-		return stderrError
+		log.Fatal(stdoutError.Error())
 	}
 	if stderrError != nil {
-		log.Warning("encountered an error opening remote stderr pipe", "file: "+file, stdoutError.Error())
-		return stderrError
+		log.Fatal(stderrError.Error())
 	}
 	var resultBytes []byte
 	var waitgroup sync.WaitGroup
@@ -125,7 +105,7 @@ func transferFile(bar *mpb.Bar, file string) error {
 			readBytes, error := stdout.Read(tmp)
 			if error != nil {
 				if error != io.EOF {
-					log.Warning("encountered an error reading stdout", "file: "+file, error.Error())
+					log.Warning("encountered an error reading stdout", error.Error())
 				}
 				break
 			}
@@ -139,9 +119,9 @@ func transferFile(bar *mpb.Bar, file string) error {
 				_, tmp, _ := strings.Cut(lowerline, "sent")
 				tmp, _, _ = strings.Cut(tmp, "bytes")
 				tmp = strings.ReplaceAll(strings.TrimSpace(tmp), ".", "")
-				bytes, error := strconv.ParseInt(tmp, 10, 64)
+				bytes, error := strconv.ParseFloat(tmp, 64)
 				if error != nil {
-					log.Fatal("encountered an error converting the transferred bytes to int", "file: "+file, error.Error())
+					log.Fatal("encountered an error converting the transferred bytes to int", error.Error())
 				}
 				transferSize += bytes
 			}
@@ -171,65 +151,46 @@ func transferFile(bar *mpb.Bar, file string) error {
 	}()
 	errorBytes, stderrError := io.ReadAll(stderr)
 	if stderrError != nil {
-		log.Warning("encountered an error reading from remote stderr", "file: "+file, stdoutError.Error())
-		return stderrError
+		log.Fatal(stderrError.Error())
 	}
 	cmd.Wait()
 	error := strings.Trim(string(errorBytes), "\n")
 	if len(error) > 0 {
-		log.Warning("encountered an remote error", "file: "+file, error)
-		return errors.New(error)
+		log.Fatal(error)
 	}
 	waitgroup.Wait()
 	stdout.Close()
 	stderr.Close()
-	return nil
+	return true
 }
 
-func getTargetLocation(sourceFile string) string {
-	if sourceFile == settings.Source {
-		return filepath.Join(settings.Target, filepath.Base(sourceFile))
+func getTargetLocation(source string) string {
+	if source == settings.Source {
+		return filepath.Join(settings.Target, filepath.Base(source))
 	}
-	source, _ := strings.CutSuffix(settings.Source, "/*")
-	if settings.SourceIsRemote() {
-		_, source, _ = strings.Cut(source, ":")
-		source = filepath.Dir(source)
-	}
-	return filepath.Join(settings.Target, strings.Replace(sourceFile, source, "", 1))
+	return filepath.Join(settings.Target, strings.Replace(source, filepath.Dir(settings.Source), "", 1))
 }
 
 func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max int64, priority int, total bool) *mpb.Bar {
-	_, green, magenta, yellow, blue := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow), color.New(color.FgBlue)
+	red, green, magenta, yellow := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow)
 	barstyle := mpb.BarStyle().Lbound("").Filler("󰝤").Tip("").Padding(" ").Rbound("")
 	defaultBarPrepend := mpb.PrependDecorators(
 		decor.Name("[info] > "),
 		decor.OnCompleteMeta(
 			decor.OnComplete(
-				decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(blue)), ""+name,
+				decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(red)), ""+name,
 			),
 			colorize(green),
 		),
-		// decor.OnAbortMeta(
-		// 	decor.OnAbort(
-		// 		decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(blue)), ""+name,
-		// 	),
-		// 	colorize(red),
-		// ),
 	)
 	defaultBarAppend := mpb.AppendDecorators(
 		decor.OnCompleteMeta(decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpaceR), colorize(yellow)),
 		decor.OnComplete(
-			decor.Name("󰇙", decor.WCSyncSpaceR), "",
+			decor.Name("|", decor.WCSyncSpaceR), "",
 		),
 		decor.OnComplete(
 			decor.Percentage(decor.WCSyncSpaceR), "",
 		),
-		// decor.OnAbort(
-		// 	decor.Name("󰇙", decor.WCSyncSpaceR), "",
-		// ),
-		// decor.OnAbort(
-		// 	decor.Percentage(decor.WCSyncSpace), "",
-		// ),
 	)
 	totalBarPrepend := mpb.PrependDecorators(
 		decor.Name("[info] > "),
@@ -268,16 +229,8 @@ func colorize(c *color.Color) func(string) string {
 	}
 }
 
-func humanizeBytes(bytes int64) (float64, string) {
-	name := "B"
-	size := float64(bytes)
-	if size >= byteToMegabyte {
-		name = "MB"
-		size = size / byteToMegabyte
+func decolorize() func(string) string {
+	return func(s string) string {
+		return ""
 	}
-	if size >= megabyteToGigabyte {
-		name = "GB"
-		size = size / megabyteToGigabyte
-	}
-	return size, name
 }

tools/scanner.go

@@ -1,97 +1,34 @@
 package tools
 
 import (
-	"errors"
-	"io"
 	"io/fs"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strconv"
-	"strings"
 	"time"
 	"velvettear/gosync/log"
 	"velvettear/gosync/settings"
 )
 
-var sourceFiles map[string]int64
+var sourceFiles []string
 
 // unexported function(s)
-func getSourceFiles() map[string]int64 {
+func getSourceFiles() []string {
 	timestamp := time.Now()
-	sourceFiles = map[string]int64{}
-	if settings.SourceIsRemote() {
-		log.Info("getting the list of remote source files...")
-		error := fillRemoteSourceFiles()
-		if error != nil {
-			log.Fatal("encountered an error getting the list of remote source files", error.Error())
-		}
+	stats, error := os.Stat(settings.Source)
+	if error != nil {
+		log.Error("encountered an error getting the stats for the source", error.Error())
+	}
+	if stats.IsDir() {
+		log.Info("scanning source...", settings.Source)
+		filepath.WalkDir(settings.Source, fillSourceFiles)
+		log.InfoTimed("found "+strconv.Itoa(len(sourceFiles))+" source files", timestamp.UnixMilli())
 	} else {
-		source, _ := strings.CutSuffix(settings.Source, "/*")
-		stats, error := os.Stat(source)
-		if error != nil {
-			log.Fatal("encountered an error getting stats for source", error.Error())
-		}
-		if stats.IsDir() {
-			log.Info("scanning source...", source)
-			filepath.WalkDir(source, fillSourceFiles)
-		} else {
-			sourceFiles[settings.Source] = stats.Size()
-		}
+		sourceFiles = append(sourceFiles, settings.Source)
 	}
-	log.InfoTimed("found "+strconv.Itoa(len(sourceFiles))+" source files", timestamp.UnixMilli())
 	return sourceFiles
 }
 
-func fillRemoteSourceFiles() error {
-	var arguments []string
-	if len(settings.Password) > 0 {
-		arguments = append(arguments, "-p", settings.Password)
-	}
-	arguments = append(arguments, "ssh")
-	remote, path, _ := strings.Cut(settings.Source, ":")
-	path, _ = strings.CutSuffix(path, "/*")
-	if len(settings.User) > 0 {
-		remote = settings.User + "@" + remote
-	}
-	arguments = append(arguments, remote, "find", "\""+path+"\"", "-type", "f", "-exec", "du", "-b", "{}", "\\;")
-	cmd := exec.Command("sshpass", arguments...)
-	stdout, stdoutError := cmd.StdoutPipe()
-	stderr, stderrError := cmd.StderrPipe()
-	cmd.Start()
-	if stdoutError != nil {
-		return stdoutError
-	}
-	if stderrError != nil {
-		return stderrError
-	}
-	outBytes, stdoutError := io.ReadAll(stdout)
-	if stdoutError != nil {
-		return stdoutError
-	}
-	errorBytes, stderrError := io.ReadAll(stderr)
-	if stderrError != nil {
-		return stderrError
-	}
-	cmd.Wait()
-	error := strings.TrimSpace(string(errorBytes))
-	if len(error) > 0 {
-		return errors.New(error)
-	}
-	for _, line := range strings.Split(string(outBytes), "\n") {
-		if !strings.Contains(line, "\t") {
-			continue
-		}
-		parts := strings.Split(line, "\t")
-		size, error := strconv.ParseInt(parts[0], 10, 64)
-		if error != nil {
-			log.Warning("encountered an error getting the file size for file '"+path+"'", error.Error())
-		}
-		sourceFiles[parts[1]] = size
-	}
-	return nil
-}
-
 func fillSourceFiles(path string, dir fs.DirEntry, err error) error {
 	if err != nil {
 		return err
@@ -99,19 +36,6 @@ func fillSourceFiles(path string, dir fs.DirEntry, err error) error {
 	if dir.IsDir() {
 		return nil
 	}
-	stats, error := os.Stat(path)
-	if error != nil {
-		log.Fatal("encountered an error getting stats for file", error.Error())
-		return nil
-	}
-	sourceFiles[path] = stats.Size()
+	sourceFiles = append(sourceFiles, path)
 	return nil
 }
-
-func getSourceFileSize() int64 {
-	var total int64
-	for _, size := range sourceFiles {
-		total += size
-	}
-	return total
-}

tools/ssh.go

@@ -11,12 +11,7 @@ import (
 
 // exported function(s)
 func TestConnection() error {
-	var remote string
-	if settings.SourceIsRemote() {
-		remote, _, _ = strings.Cut(settings.Source, ":")
-	} else if settings.TargetIsRemote() {
-		remote, _, _ = strings.Cut(settings.Target, ":")
-	} else {
+	if !settings.TargetIsRemote() {
 		return nil
 	}
 	var arguments []string
@@ -24,10 +19,11 @@ func TestConnection() error {
 		arguments = append(arguments, "-p", settings.Password)
 	}
 	arguments = append(arguments, "ssh")
+	target, _, _ := strings.Cut(settings.Target, ":")
 	if len(settings.User) > 0 {
-		remote = settings.User + "@" + remote
+		target = settings.User + "@" + target
 	}
-	arguments = append(arguments, remote, "exit")
+	arguments = append(arguments, target, "'exit'")
 	cmd := exec.Command("sshpass", arguments...)
 	stdout, stdoutError := cmd.StdoutPipe()
 	stderr, stderrError := cmd.StderrPipe()