velvettear/gosync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

initial commit

Browse source
This commit is contained in:
Daniel Sommer 2023-09-07 15:20:19 +02:00
commit 213db984ab
15 changed files with 718 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
__debug_bin
*.sqlite*

42
.vscode/launch.json vendored Normal file
View file

@ -0,0 +1,42 @@
{
"version": "0.0.1",
"configurations": [
{
"name": "gosync",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"/home/velvettear/downloads/music",
"192.168.1.11:/tmp",
"--password",
"$Velvet90",
"--concurrency",
"4",
"--verbose",
],
"console": "integratedTerminal"
},
{
"name": "gosync-root",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": [
"/home/velvettear/downloads/music",
"192.168.1.11:/tmp",
"--user",
"velvettear",
"--password",
"$Velvet90",
"--concurrency",
"4",
"--verbose",
],
"asRoot": true,
"console": "integratedTerminal"
}
]
}

3
.vscode/settings.json vendored Normal file
View file

@ -0,0 +1,3 @@
{
"sqlite.logLevel": "DEBUG"
}

20
LICENSE.md Normal file
View file

@ -0,0 +1,20 @@
# MIT License
**Copyright (c) 2022 Daniel Sommer \<daniel.sommer@velvettear.de\>**
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:
The above copyright notice and this permission notice (including the next
paragraph) shall be included in all copies or substantial portions of the
Software.
**THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.**

22
README.md Normal file
View file

@ -0,0 +1,22 @@
# gosync
a simple wrapper for concurrent rsync processes written in golang
## requirements
- [rsync](https://linux.die.net/man/1/rsync)
- [sshpass](https://linux.die.net/man/1/sshpass)
- [nerd fonts](https://www.nerdfonts.com)
## run
`gosync [source] [target] (options)`
### options
| short | long | description |
| ----- | ------------- | ------------------------------------------- |
| -u | --user | set user for ssh / rsync |
| -p | --password | set password for ssh / rsync |
| -c | --concurrency | set limit for concurrent rsync processes |
| -v | --verbose | enable verbose / debug output |

16
go.mod Normal file
View file

@ -0,0 +1,16 @@
module velvettear/gosync
go 1.21.0
require github.com/vbauerster/mpb/v8 v8.6.1
require (
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/fatih/color v1.15.0
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
golang.org/x/sys v0.12.0 // indirect
)

21
go.sum Normal file
View file

@ -0,0 +1,21 @@
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/vbauerster/mpb/v8 v8.6.1 h1:XbBpIbJxJOO9yMcKPpI4oEFPW6tLAptefNQJNcGWri8=
github.com/vbauerster/mpb/v8 v8.6.1/go.mod h1:S0tuIjikxlLxCeNijNhwAuD/BB3UE/d2nygG8SOldk0=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

BIN
gosync Executable file
View file

Binary file not shown.

99
log/log.go Normal file
View file

@ -0,0 +1,99 @@
package log
import (
"fmt"
"os"
"strconv"
"strings"
"time"
)
const LEVEL_DEBUG = 0
const LEVEL_INFO = 1
const LEVEL_WARNING = 2
const LEVEL_ERROR = 3
const LEVEL_FATAL = 4
var logLevel = 0
// exported functions
func SetLogLevel(level int) {
logLevel = level
}
func Debug(message string, extras ...string) {
DebugTimed(message, -1, extras...)
}
func DebugTimed(message string, timestamp int64, extras ...string) {
trace(LEVEL_DEBUG, timestamp, message, extras...)
}
func Info(message string, extras ...string) {
InfoTimed(message, -1, extras...)
}
func InfoTimed(message string, timestamp int64, extras ...string) {
trace(LEVEL_INFO, timestamp, message, extras...)
}
func Warning(message string, extras ...string) {
WarningTimed(message, -1, extras...)
}
func WarningTimed(message string, timestamp int64, extras ...string) {
trace(LEVEL_WARNING, timestamp, message, extras...)
}
func Error(message string, extras ...string) {
ErrorTimed(message, -1, extras...)
}
func ErrorTimed(message string, timestamp int64, extras ...string) {
trace(LEVEL_ERROR, -1, message, extras...)
}
func Fatal(message string, extras ...string) {
FatalTimed(message, -1, extras...)
}
func FatalTimed(message string, timestamp int64, extras ...string) {
trace(LEVEL_FATAL, timestamp, message, extras...)
trace(LEVEL_FATAL, -1, "exiting...")
os.Exit(1)
}
// unexported functions
func trace(level int, timestamp int64, message string, extras ...string) {
if len(message) == 0 || level < logLevel {
return
}
suffix := strings.Join(extras, " | ")
if len(suffix) > 0 {
message += " (" + suffix + ")"
}
if timestamp >= 0 {
message += " [" + strconv.Itoa(int(time.Now().UnixMilli()-timestamp)) + "ms" + "]"
}
fmt.Println(buildLogMessage(getPrefixForLogLevel(level), message))
}
func getPrefixForLogLevel(level int) string {
switch level {
case LEVEL_FATAL:
return "fatal"
case LEVEL_ERROR:
return "error"
case LEVEL_WARNING:
return "warning"
case LEVEL_INFO:
return "info"
default:
return "debug"
}
}
func buildLogMessage(prefix string, message string) string {
return "[" + prefix + "] > " + message
}

26
main.go Normal file
View file

@ -0,0 +1,26 @@
package main
import (
"os"
"time"
"velvettear/gosync/log"
"velvettear/gosync/settings"
"velvettear/gosync/tools"
)
func main() {
timestamp := time.Now()
settings.Initialize()
log.Info("starting gosync...")
error := tools.TestConnection()
if error != nil {
log.Fatal("encountered an error connecting to the remove target", error.Error())
}
tools.Transfer()
exit(timestamp, 0)
}
func exit(timestamp time.Time, code int) {
log.InfoTimed("gosync finished - exiting...", timestamp.UnixMilli())
os.Exit(code)
}

83
settings/arguments.go Normal file
View file

@ -0,0 +1,83 @@
package settings
import (
"os"
"runtime"
"strconv"
"strings"
"velvettear/gosync/log"
)
// exported function(s)
func Initialize() {
os.Args = os.Args[1:]
if len(os.Args) < 2 {
log.Fatal("error: missing arguments")
}
var arguments []string
for index, arg := range os.Args {
switch strings.ToLower(arg) {
case "-v":
fallthrough
case "--verbose":
setVerbose(true)
case "-c":
fallthrough
case "--concurrency":
var concurrency int
tmpIndex := index + 1
if tmpIndex < len(os.Args) {
tmp, error := strconv.Atoi(os.Args[tmpIndex])
if error == nil {
concurrency = tmp
}
}
if concurrency == 0 {
concurrency = runtime.NumCPU()
}
setConcurrency(concurrency)
case "-p":
fallthrough
case "--password":
tmpIndex := index + 1
if index > len(os.Args) {
break
}
setPassword(os.Args[tmpIndex])
case "-u":
fallthrough
case "--user":
tmpIndex := index + 1
if index > len(os.Args) {
break
}
setUser(os.Args[tmpIndex])
default:
arguments = append(arguments, arg)
}
}
setSource(arguments[0])
setTarget(arguments[1])
if Concurrency == 0 {
setConcurrency(runtime.NumCPU())
}
_, error := os.Stat(Source)
if os.IsNotExist(error) {
log.Fatal("given source does not exist", Source)
}
if !Verbose {
setVerbose(false)
}
}
// unexported function(s)
func removeArgument(index int) {
removeArguments(index, 0, 0)
}
func removeArguments(index int, before int, after int) {
// derp := index - 1 - before
copyArgs := os.Args[0 : index-before]
copyArgs = append(copyArgs, os.Args[index+1+after:]...)
os.Args = copyArgs
}

56
settings/variables.go Normal file
View file

@ -0,0 +1,56 @@
package settings
import (
"strconv"
"strings"
"velvettear/gosync/log"
)
// exported variable(s)
var Verbose bool
var Source string
var Target string
var Concurrency int
var Password string
var User string
// exported function(s)
func TargetIsRemote() bool {
return strings.Contains(Target, ":")
}
// unexported function(s)
func setVerbose(verbose bool) {
Verbose = verbose
if Verbose {
log.SetLogLevel(0)
} else {
log.SetLogLevel(1)
}
log.Debug("set verbose flag", strconv.FormatBool(Verbose))
}
func setSource(source string) {
Source = source
log.Debug("set source", Source)
}
func setTarget(target string) {
Target = target
log.Debug("set target", Target)
}
func setConcurrency(concurrency int) {
Concurrency = concurrency
log.Debug("set concurrency", strconv.Itoa(Concurrency))
}
func setPassword(password string) {
Password = password
log.Debug("set password", Password)
}
func setUser(user string) {
User = user
log.Debug("set user", User)
}

232
tools/rsync.go Normal file
View file

@ -0,0 +1,232 @@
package tools
import (
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"velvettear/gosync/log"
"velvettear/gosync/settings"
"github.com/fatih/color"
"github.com/vbauerster/mpb/v8"
"github.com/vbauerster/mpb/v8/decor"
)
var transferSize float64
// exported function(s)
func Transfer() {
transferSize = 0
sourcefiles := getSourceFiles()
sourcefilesCount := len(sourcefiles)
var waitgroup sync.WaitGroup
waitgroup.Add(sourcefilesCount)
barcontainer := mpb.New(
mpb.WithWaitGroup(&waitgroup),
)
timestamp := time.Now()
counter := 0
totalbar := createProgressBar(barcontainer, "Total", int64(0), int64(sourcefilesCount), sourcefilesCount+1, true)
channel := make(chan struct{}, settings.Concurrency)
for index, file := range sourcefiles {
channel <- struct{}{}
if index > 0 {
time.Sleep(100 * time.Millisecond)
}
go func(index int, file string) {
defer waitgroup.Done()
stats, error := os.Stat(file)
if error != nil {
log.Warning("encountered an error getting file size", error.Error())
return
}
bar := createProgressBar(barcontainer, filepath.Base(file), stats.Size(), int64(100), index, false)
if transferFile(bar, file) {
counter++
}
totalbar.Increment()
<-channel
}(index, file)
}
barcontainer.Wait()
timeDifference := time.Since(timestamp)
transferSpeedName := "bytes/sec"
transferSpeed := transferSize / timeDifference.Seconds()
if transferSpeed > 1048576 {
transferSpeed = transferSpeed / 1048576
transferSpeedName = "mb/s"
}
transferSizeName := "bytes"
if transferSize > 1048576 {
transferSize = transferSize / 1048576
transferSizeName = "mb"
}
log.InfoTimed("transferred "+strconv.Itoa(counter)+" files, "+strconv.Itoa(int(transferSize))+" "+transferSizeName+" ("+strconv.FormatFloat(transferSpeed, 'f', 2, 64)+" "+transferSpeedName+")", timestamp.UnixMilli())
}
// unexported function(s)
func transferFile(bar *mpb.Bar, file string) bool {
target := getTargetLocation(file)
var arguments []string
if len(settings.Password) > 0 {
arguments = append(arguments, "-p", settings.Password)
}
arguments = append(arguments, "rsync", "-avz", "-mkpath", file)
if len(settings.User) > 0 {
target = settings.User + "@" + target
}
arguments = append(arguments, target, "--progress")
cmd := exec.Command("sshpass", arguments...)
stdout, stdoutError := cmd.StdoutPipe()
stderr, stderrError := cmd.StderrPipe()
cmd.Start()
if stdoutError != nil {
log.Fatal(stdoutError.Error())
}
if stderrError != nil {
log.Fatal(stderrError.Error())
}
var resultBytes []byte
var waitgroup sync.WaitGroup
waitgroup.Add(1)
go func() {
defer waitgroup.Done()
for {
tmp := make([]byte, 1024)
readBytes, error := stdout.Read(tmp)
if error != nil {
if error != io.EOF {
log.Warning("encountered an error reading stdout", error.Error())
}
break
}
if readBytes == 0 {
break
}
resultBytes = append(resultBytes, tmp...)
line := string(tmp)
lowerline := strings.ToLower(line)
if strings.Contains(lowerline, "sent") && strings.Contains(lowerline, "received") && strings.Contains(lowerline, "bytes") {
_, tmp, _ := strings.Cut(lowerline, "sent")
tmp, _, _ = strings.Cut(tmp, "bytes")
tmp = strings.ReplaceAll(strings.TrimSpace(tmp), ".", "")
bytes, error := strconv.ParseFloat(tmp, 64)
if error != nil {
log.Fatal("encountered an error converting the transferred bytes to int", error.Error())
}
transferSize += bytes
}
if strings.Contains(lowerline, "total size is") && strings.Contains(lowerline, "speedup is") {
bar.SetCurrent(100)
break
}
cut := strings.Index(line, "%")
if cut <= 0 {
continue
}
line = line[:cut]
var percent string
for index := len(line) - 1; index > 0; index-- {
char := string(line[index])
if char == " " {
break
}
percent = char + percent
}
value, error := strconv.Atoi(percent)
if error != nil {
continue
}
bar.SetCurrent(int64(value))
}
}()
errorBytes, stderrError := io.ReadAll(stderr)
if stderrError != nil {
log.Fatal(stderrError.Error())
}
cmd.Wait()
error := strings.Trim(string(errorBytes), "\n")
if len(error) > 0 {
log.Fatal(error)
}
waitgroup.Wait()
stdout.Close()
stderr.Close()
return true
}
func getTargetLocation(source string) string {
if source == settings.Source {
return filepath.Join(settings.Target, filepath.Base(source))
}
return filepath.Join(settings.Target, strings.Replace(source, filepath.Dir(settings.Source), "", 1))
}
func createProgressBar(barcontainer *mpb.Progress, name string, size int64, max int64, priority int, total bool) *mpb.Bar {
red, green, magenta, yellow := color.New(color.FgRed), color.New(color.FgGreen), color.New(color.FgMagenta), color.New(color.FgYellow)
barstyle := mpb.BarStyle().Lbound("").Filler("󰝤").Tip("").Padding(" ").Rbound("")
defaultBarPrepend := mpb.PrependDecorators(
decor.Name("[info] > "),
decor.OnCompleteMeta(
decor.OnComplete(
decor.Meta(decor.Name(name, decor.WCSyncSpaceR), colorize(red)), ""+name,
),
colorize(green),
),
)
defaultBarAppend := mpb.AppendDecorators(
decor.OnCompleteMeta(decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpaceR), colorize(yellow)),
decor.OnComplete(
decor.Name("|", decor.WCSyncSpaceR), "",
),
decor.OnComplete(
decor.Percentage(decor.WCSyncSpaceR), "",
),
)
totalBarPrepend := mpb.PrependDecorators(
decor.Name("[info] > "),
decor.OnCompleteMeta(
decor.OnComplete(
decor.Meta(decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}), colorize(yellow)), ""+name,
),
colorize(magenta),
),
decor.CountersNoUnit("%d / %d"),
)
if total {
return barcontainer.New(
max,
barstyle,
mpb.BarPriority(priority),
mpb.BarFillerClearOnComplete(),
totalBarPrepend,
defaultBarAppend,
)
} else {
return barcontainer.New(
max,
barstyle,
mpb.BarPriority(priority),
mpb.BarFillerClearOnComplete(),
defaultBarPrepend,
defaultBarAppend,
)
}
}
func colorize(c *color.Color) func(string) string {
return func(s string) string {
return c.Sprint(s)
}
}
func decolorize() func(string) string {
return func(s string) string {
return ""
}
}

41
tools/scanner.go Normal file
View file

@ -0,0 +1,41 @@
package tools
import (
"io/fs"
"os"
"path/filepath"
"strconv"
"time"
"velvettear/gosync/log"
"velvettear/gosync/settings"
)
var sourceFiles []string
// unexported function(s)
func getSourceFiles() []string {
timestamp := time.Now()
stats, error := os.Stat(settings.Source)
if error != nil {
log.Error("encountered an error getting the stats for the source", error.Error())
}
if stats.IsDir() {
log.Info("scanning source...", settings.Source)
filepath.WalkDir(settings.Source, fillSourceFiles)
log.InfoTimed("found "+strconv.Itoa(len(sourceFiles))+" source files", timestamp.UnixMilli())
} else {
sourceFiles = append(sourceFiles, settings.Source)
}
return sourceFiles
}
func fillSourceFiles(path string, dir fs.DirEntry, err error) error {
if err != nil {
return err
}
if dir.IsDir() {
return nil
}
sourceFiles = append(sourceFiles, path)
return nil
}

55
tools/ssh.go Normal file
View file

@ -0,0 +1,55 @@
package tools
import (
"errors"
"io"
"os/exec"
"strings"
"velvettear/gosync/log"
"velvettear/gosync/settings"
)
// exported function(s)
func TestConnection() error {
if !settings.TargetIsRemote() {
return nil
}
if len(settings.Password) == 0 {
log.Warning("target is a remote host and no password is set, make sure passwordless login is configured")
}
var arguments []string
if len(settings.Password) > 0 {
arguments = append(arguments, "-p", settings.Password)
}
arguments = append(arguments, "ssh")
target, _, _ := strings.Cut(settings.Target, ":")
if len(settings.User) > 0 {
target = settings.User + "@" + target
}
arguments = append(arguments, target, "'exit'")
cmd := exec.Command("sshpass", arguments...)
stdout, stdoutError := cmd.StdoutPipe()
stderr, stderrError := cmd.StderrPipe()
cmd.Start()
if stdoutError != nil {
return stdoutError
}
if stderrError != nil {
return stderrError
}
_, stdoutError = io.ReadAll(stdout)
if stdoutError != nil {
return stdoutError
}
errorBytes, stderrError := io.ReadAll(stderr)
if stderrError != nil {
return stderrError
}
cmd.Wait()
error := strings.TrimSpace(string(errorBytes))
if len(error) > 0 {
return errors.New(error)
}
return nil
}