commit 213db984abfbf97e8ff9d67b34586bf016248dca Author: velvettear Date: Thu Sep 7 15:20:19 2023 +0200 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b8e41cc --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__debug_bin +*.sqlite* diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..490939f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,42 @@ +{ + "version": "0.0.1", + "configurations": [ + { + "name": "gosync", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/main.go", + "args": [ + "/home/velvettear/downloads/music", + "192.168.1.11:/tmp", + "--password", + "$Velvet90", + "--concurrency", + "4", + "--verbose", + ], + "console": "integratedTerminal" + }, + { + "name": "gosync-root", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/main.go", + "args": [ + "/home/velvettear/downloads/music", + "192.168.1.11:/tmp", + "--user", + "velvettear", + "--password", + "$Velvet90", + "--concurrency", + "4", + "--verbose", + ], + "asRoot": true, + "console": "integratedTerminal" + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..c65ede2 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "sqlite.logLevel": "DEBUG" +} \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..d342365 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,20 @@ +# MIT License +**Copyright (c) 2022 Daniel Sommer \** + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the next +paragraph) shall be included in all copies or substantial portions of the +Software. + +**THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF +OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.** \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..fa53692 --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# gosync + +a simple wrapper for concurrent rsync processes written in golang + +## requirements + +- [rsync](https://linux.die.net/man/1/rsync) +- [sshpass](https://linux.die.net/man/1/sshpass) +- [nerd fonts](https://www.nerdfonts.com) + +## run + +`gosync [source] [target] (options)` + +### options + +| short | long | description | +| ----- | ------------- | ------------------------------------------- | +| -u | --user | set user for ssh / rsync | +| -p | --password | set password for ssh / rsync | +| -c | --concurrency | set limit for concurrent rsync processes | +| -v | --verbose | enable verbose / debug output | \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..626e17d --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module velvettear/gosync + +go 1.21.0 + +require github.com/vbauerster/mpb/v8 v8.6.1 + +require ( + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect + github.com/fatih/color v1.15.0 + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + golang.org/x/sys v0.12.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..772e75e --- /dev/null +++ b/go.sum @@ -0,0 +1,21 @@ +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/vbauerster/mpb/v8 v8.6.1 h1:XbBpIbJxJOO9yMcKPpI4oEFPW6tLAptefNQJNcGWri8= +github.com/vbauerster/mpb/v8 v8.6.1/go.mod h1:S0tuIjikxlLxCeNijNhwAuD/BB3UE/d2nygG8SOldk0= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/gosync b/gosync new file mode 100755 index 0000000..6e06211 Binary files /dev/null and b/gosync differ diff --git a/log/log.go b/log/log.go new file mode 100644 index 0000000..91e0cbb --- /dev/null +++ b/log/log.go @@ -0,0 +1,99 @@ +package log + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +const LEVEL_DEBUG = 0 +const LEVEL_INFO = 1 +const LEVEL_WARNING = 2 +const LEVEL_ERROR = 3 +const LEVEL_FATAL = 4 + +var logLevel = 0 + +// exported functions +func SetLogLevel(level int) { + logLevel = level +} + +func Debug(message string, extras ...string) { + DebugTimed(message, -1, extras...) +} + +func DebugTimed(message string, timestamp int64, extras ...string) { + trace(LEVEL_DEBUG, timestamp, message, extras...) +} + +func Info(message string, extras ...string) { + InfoTimed(message, -1, extras...) +} + +func InfoTimed(message string, timestamp int64, extras ...string) { + trace(LEVEL_INFO, timestamp, message, extras...) +} + +func Warning(message string, extras ...string) { + WarningTimed(message, -1, extras...) +} + +func WarningTimed(message string, timestamp int64, extras ...string) { + trace(LEVEL_WARNING, timestamp, message, extras...) +} + +func Error(message string, extras ...string) { + ErrorTimed(message, -1, extras...) +} + +func ErrorTimed(message string, timestamp int64, extras ...string) { + trace(LEVEL_ERROR, -1, message, extras...) +} + +func Fatal(message string, extras ...string) { + FatalTimed(message, -1, extras...) +} + +func FatalTimed(message string, timestamp int64, extras ...string) { + trace(LEVEL_FATAL, timestamp, message, extras...) + trace(LEVEL_FATAL, -1, "exiting...") + os.Exit(1) +} + +// unexported functions +func trace(level int, timestamp int64, message string, extras ...string) { + if len(message) == 0 || level < logLevel { + return + } + suffix := strings.Join(extras, " | ") + if len(suffix) > 0 { + message += " (" + suffix + ")" + } + if timestamp >= 0 { + + message += " [" + strconv.Itoa(int(time.Now().UnixMilli()-timestamp)) + "ms" + "]" + } + fmt.Println(buildLogMessage(getPrefixForLogLevel(level), message)) +} + +func getPrefixForLogLevel(level int) string { + switch level { + case LEVEL_FATAL: + return "fatal" + case LEVEL_ERROR: + return "error" + case LEVEL_WARNING: + return "warning" + case LEVEL_INFO: + return "info" + default: + return "debug" + } +} + +func buildLogMessage(prefix string, message string) string { + return "[" + prefix + "] > " + message +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..b3b4bdd --- /dev/null +++ b/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "os" + "time" + "velvettear/gosync/log" + "velvettear/gosync/settings" + "velvettear/gosync/tools" +) + +func main() { + timestamp := time.Now() + settings.Initialize() + log.Info("starting gosync...") + error := tools.TestConnection() + if error != nil { + log.Fatal("encountered an error connecting to the remove target", error.Error()) + } + tools.Transfer() + exit(timestamp, 0) +} + +func exit(timestamp time.Time, code int) { + log.InfoTimed("gosync finished - exiting...", timestamp.UnixMilli()) + os.Exit(code) +} diff --git a/settings/arguments.go b/settings/arguments.go new file mode 100644 index 0000000..3dff27e --- /dev/null +++ b/settings/arguments.go @@ -0,0 +1,83 @@ +package settings + +import ( + "os" + "runtime" + "strconv" + "strings" + "velvettear/gosync/log" +) + +// exported function(s) +func Initialize() { + os.Args = os.Args[1:] + if len(os.Args) < 2 { + log.Fatal("error: missing arguments") + } + var arguments []string + for index, arg := range os.Args { + switch strings.ToLower(arg) { + case "-v": + fallthrough + case "--verbose": + setVerbose(true) + case "-c": + fallthrough + case "--concurrency": + var concurrency int + tmpIndex := index + 1 + if tmpIndex < len(os.Args) { + tmp, error := strconv.Atoi(os.Args[tmpIndex]) + if error == nil { + concurrency = tmp + } + } + if concurrency == 0 { + concurrency = runtime.NumCPU() + } + setConcurrency(concurrency) + case "-p": + fallthrough + case "--password": + tmpIndex := index + 1 + if index > len(os.Args) { + break + } + setPassword(os.Args[tmpIndex]) + case "-u": + fallthrough + case "--user": + tmpIndex := index + 1 + if index > len(os.Args) { + break + } + setUser(os.Args[tmpIndex]) + default: + arguments = append(arguments, arg) + } + } + setSource(arguments[0]) + setTarget(arguments[1]) + if Concurrency == 0 { + setConcurrency(runtime.NumCPU()) + } + _, error := os.Stat(Source) + if os.IsNotExist(error) { + log.Fatal("given source does not exist", Source) + } + if !Verbose { + setVerbose(false) + } +} + +// unexported function(s) +func removeArgument(index int) { + removeArguments(index, 0, 0) +} + +func removeArguments(index int, before int, after int) { + // derp := index - 1 - before + copyArgs := os.Args[0 : index-before] + copyArgs = append(copyArgs, os.Args[index+1+after:]...) + os.Args = copyArgs +} diff --git a/settings/variables.go b/settings/variables.go new file mode 100644 index 0000000..e8d8645 --- /dev/null +++ b/settings/variables.go @@ -0,0 +1,56 @@ +package settings + +import ( + "strconv" + "strings" + "velvettear/gosync/log" +) + +// exported variable(s) +var Verbose bool +var Source string +var Target string +var Concurrency int +var Password string +var User string + +// exported function(s) +func TargetIsRemote() bool { + return strings.Contains(Target, ":") +} + +// unexported function(s) +func setVerbose(verbose bool) { + Verbose = verbose + if Verbose { + log.SetLogLevel(0) + } else { + log.SetLogLevel(1) + } + log.Debug("set verbose flag", strconv.FormatBool(Verbose)) +} + +func setSource(source string) { + Source = source + log.Debug("set source", Source) +} + +func setTarget(target string) { + Target = target + log.Debug("set target", Target) +} + +func setConcurrency(concurrency int) { + Concurrency = concurrency + log.Debug("set concurrency", strconv.Itoa(Concurrency)) +} + +func setPassword(password string) { + Password = password + log.Debug("set password", Password) +} + +func setUser(user string) { + User = user + log.Debug("set user", User) +} diff --git a/tools/rsync.go b/tools/rsync.go new file mode 100644 index 0000000..1595989 --- /dev/null +++ b/tools/rsync.go @@ -0,0 +1,232 @@ +package tools + +import ( + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + "velvettear/gosync/log" + "velvettear/gosync/settings" + + "github.com/fatih/color" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" +) + +var transferSize float64 + +// exported function(s) +func Transfer() { + transferSize = 0 + sourcefiles := getSourceFiles() + sourcefilesCount := len(sourcefiles) + var waitgroup sync.WaitGroup + waitgroup.Add(sourcefilesCount) + barcontainer := mpb.New( + mpb.WithWaitGroup(&waitgroup), + ) + timestamp := time.Now() + counter := 0 + totalbar := createProgressBar(barcontainer, "Total", int64(0), int64(sourcefilesCount), sourcefilesCount+1, true) + channel := make(chan struct{}, settings.Concurrency) + for index, file := range sourcefiles { + channel <- struct{}{} + if index > 0 { + time.Sleep(100 * time.Millisecond) + } + go func(index int, file string) { + defer waitgroup.Done() + stats, error := os.Stat(file) + if error != nil { + log.Warning("encountered an error getting file size", error.Error()) + return + } + bar := createProgressBar(barcontainer, filepath.Base(file), stats.Size(), int64(100), index, false) + if transferFile(bar, file) { + counter++ + } + totalbar.Increment() + <-channel + }(index, file) + } + barcontainer.Wait() + timeDifference := time.Since(timestamp) + transferSpeedName := "bytes/sec" + transferSpeed := transferSize / timeDifference.Seconds() + if transferSpeed > 1048576 { + transferSpeed = transferSpeed / 1048576 + transferSpeedName = "mb/s" + } + transferSizeName := "bytes" + if transferSize > 1048576 { + transferSize = transferSize / 1048576 + transferSizeName = "mb" + } + log.InfoTimed("transferred "+strconv.Itoa(counter)+" files, "+strconv.Itoa(int(transferSize))+" "+transferSizeName+" ("+strconv.FormatFloat(transferSpeed, 'f', 2, 64)+" "+transferSpeedName+")", timestamp.UnixMilli()) +} + +// unexported function(s) +func transferFile(bar *mpb.Bar, file string) bool { + target := getTargetLocation(file) + var arguments []string + if len(settings.Password) > 0 { + arguments = append(arguments, "-p", settings.Password) + } + arguments = append(arguments, "rsync", "-avz", "-mkpath", file) + if len(settings.User) > 0 { + target = settings.User + "@" + target + } + arguments = append(arguments, target, "--progress") + cmd := exec.Command("sshpass", arguments...) + stdout, stdoutError := cmd.StdoutPipe() + stderr, stderrError := cmd.StderrPipe() + cmd.Start() + if stdoutError != nil { + log.Fatal(stdoutError.Error()) + } + if stderrError != nil { + log.Fatal(stderrError.Error()) + } + var resultBytes []byte + var waitgroup sync.WaitGroup + waitgroup.Add(1) + go func() { + defer waitgroup.Done() + for { + tmp := make([]byte, 1024) + readBytes, error := stdout.Read(tmp) + if error != nil { + if error != io.EOF { + log.Warning("encountered an error reading stdout", error.Error()) + } + break + } + if readBytes == 0 { + break + } + resultBytes = append(resultBytes, tmp...) + line := string(tmp) + lowerline := strings.ToLower(line) + if strings.Contains(lowerline, "sent") && strings.Contains(lowerline, "received") && strings.Contains(lowerline, "bytes") { + _, tmp, _ := strings.Cut(lowerline, "sent") + tmp, _, _ = strings.Cut(tmp, "bytes") + tmp = strings.ReplaceAll(strings.TrimSpace(tmp), ".", "") + bytes, error := strconv.ParseFloat(tmp, 64) + if error != nil { + log.Fatal("encountered an error converting the transferred bytes to int", error.Error()) + } + transferSize += bytes + } + if strings.Contains(lowerline, "total size is") && strings.Contains(lowerline, "speedup is") { + bar.SetCurrent(100) + break + } + cut := strings.Index(line, "%") + if cut <= 0 { + continue + } + line = line[:cut] + var percent string + for index := len(line) - 1; index > 0; index-- { + char := string(line[index]) + if char == " " { + break + } + percent = char + percent + } + value, error := strconv.Atoi(percent) + if error != nil { + continue + } + bar.SetCurrent(int64(value)) + } + }() + errorBytes, stderrError := io.ReadAll(stderr) + if stderrError != nil { + log.Fatal(stderrError.Error()) + } + cmd.Wait() + error := strings.Trim(string(errorBytes), "\n") + if len(error) > 0 { + log.Fatal(error) + } + waitgroup.Wait() + stdout.Close() + stderr.Close() + return true +} + +func getTargetLocation(source string) string { + if source == settings.Source { + return filepath.Join(settings.Target, filepath.Base(source)) + } + return filepath.Join(settings.Target, strings.Replace(source, filepath.Dir(settings.Source), "", 1)) +} + +func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max int64, priority int, total bool) *mpb.Bar { + red, green, magenta, yellow := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow) + barstyle := mpb.BarStyle().Lbound("").Filler("󰝤").Tip("").Padding(" ").Rbound("") + defaultBarPrepend := mpb.PrependDecorators( + decor.Name("[info] > "), + decor.OnCompleteMeta( + decor.OnComplete( + decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(red)), ""+name, + ), + colorize(green), + ), + ) + defaultBarAppend := mpb.AppendDecorators( + decor.OnCompleteMeta(decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpaceR), colorize(yellow)), + decor.OnComplete( + decor.Name("|", decor.WCSyncSpaceR), "", + ), + decor.OnComplete( + decor.Percentage(decor.WCSyncSpaceR), "", + ), + ) + totalBarPrepend := mpb.PrependDecorators( + decor.Name("[info] > "), + decor.OnCompleteMeta( + decor.OnComplete( + decor.Meta(decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}), colorize(yellow)), ""+name, + ), + colorize(magenta), + ), + decor.CountersNoUnit("%d / %d"), + ) + if total { + return barcontainer.New( + max, + barstyle, + mpb.BarPriority(priority), + mpb.BarFillerClearOnComplete(), + totalBarPrepend, + defaultBarAppend, + ) + } else { + return barcontainer.New( + max, + barstyle, + mpb.BarPriority(priority), + mpb.BarFillerClearOnComplete(), + defaultBarPrepend, + defaultBarAppend, + ) + } +} + +func colorize(c *color.Color) func(string) string { + return func(s string) string { + return c.Sprint(s) + } +} + +func decolorize() func(string) string { + return func(s string) string { + return "" + } +} diff --git a/tools/scanner.go b/tools/scanner.go new file mode 100644 index 0000000..2118813 --- /dev/null +++ b/tools/scanner.go @@ -0,0 +1,41 @@ +package tools + +import ( + "io/fs" + "os" + "path/filepath" + "strconv" + "time" + "velvettear/gosync/log" + "velvettear/gosync/settings" +) + +var sourceFiles []string + +// unexported function(s) +func getSourceFiles() []string { + timestamp := time.Now() + stats, error := os.Stat(settings.Source) + if error != nil { + log.Error("encountered an error getting the stats for the source", error.Error()) + } + if stats.IsDir() { + log.Info("scanning source...", settings.Source) + filepath.WalkDir(settings.Source, fillSourceFiles) + log.InfoTimed("found "+strconv.Itoa(len(sourceFiles))+" source files", timestamp.UnixMilli()) + } else { + sourceFiles = append(sourceFiles, settings.Source) + } + return sourceFiles +} + +func fillSourceFiles(path string, dir fs.DirEntry, err error) error { + if err != nil { + return err + } + if dir.IsDir() { + return nil + } + sourceFiles = append(sourceFiles, path) + return nil +} diff --git a/tools/ssh.go b/tools/ssh.go new file mode 100644 index 0000000..bff65f3 --- /dev/null +++ b/tools/ssh.go @@ -0,0 +1,55 @@ +package tools + +import ( + "errors" + "io" + + "os/exec" + "strings" + "velvettear/gosync/log" + "velvettear/gosync/settings" +) + +// exported function(s) +func TestConnection() error { + if !settings.TargetIsRemote() { + return nil + } + if len(settings.Password) == 0 { + log.Warning("target is a remote host and no password is set, make sure passwordless login is configured") + } + var arguments []string + if len(settings.Password) > 0 { + arguments = append(arguments, "-p", settings.Password) + } + arguments = append(arguments, "ssh") + target, _, _ := strings.Cut(settings.Target, ":") + if len(settings.User) > 0 { + target = settings.User + "@" + target + } + arguments = append(arguments, target, "'exit'") + cmd := exec.Command("sshpass", arguments...) + stdout, stdoutError := cmd.StdoutPipe() + stderr, stderrError := cmd.StderrPipe() + cmd.Start() + if stdoutError != nil { + return stdoutError + } + if stderrError != nil { + return stderrError + } + _, stdoutError = io.ReadAll(stdout) + if stdoutError != nil { + return stdoutError + } + errorBytes, stderrError := io.ReadAll(stderr) + if stderrError != nil { + return stderrError + } + cmd.Wait() + error := strings.TrimSpace(string(errorBytes)) + if len(error) > 0 { + return errors.New(error) + } + return nil +}