From 6b08d2807bd2257d4bb09ffdc2e55e2cb15b54ae Mon Sep 17 00:00:00 2001 From: velvettear Date: Tue, 15 Aug 2023 14:51:54 +0200 Subject: [PATCH] probably finished project --- .gitignore | 4 +- .vscode/launch.json | 7 +-- README.md | 12 ++++- files/cleaner.go | 55 +++++++++++++++++++++++ files/mover.go | 39 +++++++++++++++++ files/{finder.go => scanner.go} | 17 ++++++- go.mod | 2 +- main.go | 61 ++++++++++++++++++++++++++ prompts/prompts.go | 78 +++++++++++++++++++++++++++++++++ settings/arguments.go | 10 ++++- settings/variables.go | 14 ++++++ 11 files changed, 289 insertions(+), 10 deletions(-) create mode 100644 files/cleaner.go create mode 100644 files/mover.go rename files/{finder.go => scanner.go} (63%) create mode 100644 prompts/prompts.go diff --git a/.gitignore b/.gitignore index 9a19b49..ca5d71b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -__debug_bin -worklog +__debug_bin* +dedupe \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 4fe4729..c13369a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,10 +8,11 @@ "mode": "auto", "program": "${workspaceFolder}/main.go", "args": [ - "/tmp/nfs/music/lossless", - "/tmp/nfs/music/mp3", + "/home/velvettear/music/lossless", + "/home/velvettear/music/mp3", "-v", - "--delete" + "-m", + "/home/velvettear/music/duplicates" ] } ] diff --git a/README.md b/README.md index 2d8e194..c40480a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,13 @@ # dedupe -simple command line tool to find and move/delete duplicate audio files \ No newline at end of file +simple command line tool to find and move/delete duplicate audio files + +## run + +`dedupe [source] [comparison] (options)` + +### options + +- -d | --delete: delete duplicate files +- -m | --move: move duplicate files to specified directory +- -v | --verbose: enable verbose / debug output diff --git a/files/cleaner.go b/files/cleaner.go new file mode 100644 index 0000000..f2ebc23 --- /dev/null +++ b/files/cleaner.go @@ -0,0 +1,55 @@ +package files + +import ( + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + "velvettear/dedupe/log" +) + +var subdirectories []string + +// exported function(s) +func RemoveEmptyDirectories(directory string) { + subdirectories = nil + timestamp := time.Now() + log.Info("deleting empty subdirectories...", "directory: "+directory) + filepath.WalkDir(directory, collectSubdirectories) + sort.Slice(subdirectories, func(i, j int) bool { + return strings.Count(subdirectories[i], string(os.PathSeparator)) > strings.Count(subdirectories[j], string(os.PathSeparator)) + }) + count := 0 + for _, subdirectory := range subdirectories { + files, error := os.ReadDir(subdirectory) + if error != nil { + log.Error("encountered an error checking the content of directory '" + subdirectory) + } + if len(files) > 0 { + continue + } + error = os.Remove(subdirectory) + if error != nil { + log.Warning("encountered an error deleting directory '"+subdirectory+"'", error.Error()) + continue + } + count++ + log.Debug("deleted directory '" + subdirectory + "'") + } + log.InfoTimed("finished deleting "+strconv.Itoa(count)+" empty subdirectoies", timestamp.UnixMilli(), "directory: "+directory) +} + +// unexported function(s) +func collectSubdirectories(path string, dir fs.DirEntry, err error) error { + if err != nil { + return err + } + if !dir.IsDir() { + return nil + } + subdirectories = append(subdirectories, path) + return nil +} diff --git a/files/mover.go b/files/mover.go new file mode 100644 index 0000000..354528e --- /dev/null +++ b/files/mover.go @@ -0,0 +1,39 @@ +package files + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "time" + "velvettear/dedupe/log" + "velvettear/dedupe/settings" +) + +// exported function(s) +func MoveFile(file string) bool { + timestamp := time.Now() + targetFile := filepath.Join(settings.MoveDirectory, strings.Replace(file, settings.ComparisonDirectory, "", 1)) + targetDirectory := filepath.Dir(targetFile) + error := createTargetDirectory(targetDirectory, 0777) + if error != nil { + log.Error("encountered an error creating the directory '"+targetDirectory+"'", error.Error()) + return false + } + error = os.Rename(file, targetFile) + if error != nil { + log.Error("encountered an error moving the file '"+file+"' to '"+targetFile+"'", error.Error()) + return false + } + log.DebugTimed("moved file '"+file+"' to '"+targetFile+"'", timestamp.UnixMilli()) + return true +} + +func createTargetDirectory(directory string, permissions fs.FileMode) error { + error := os.MkdirAll(directory, permissions) + if error != nil { + return error + } + log.Debug("created directory '" + directory + "'") + return nil +} diff --git a/files/finder.go b/files/scanner.go similarity index 63% rename from files/finder.go rename to files/scanner.go index 52719e6..4bf450c 100644 --- a/files/finder.go +++ b/files/scanner.go @@ -4,11 +4,14 @@ import ( "io/fs" "path/filepath" "strconv" + "strings" "time" "velvettear/dedupe/log" "velvettear/dedupe/settings" ) +var Duplicates []string + var sourceFiles []string var comparisonFiles []string @@ -24,11 +27,21 @@ func Scan() { filepath.WalkDir(settings.ComparisonDirectory, fillComparisonFiles) log.InfoTimed("found "+strconv.Itoa(len(comparisonFiles))+" comparison files", timestamp.UnixMilli()) + timestamp = time.Now() + log.Info("comparing files...") for _, sourceFile := range sourceFiles { log.Debug("checking file", sourceFile) - sourceFileName := filepath.Base(sourceFile) - log.Debug("derp", sourceFileName) + sourceFilePath := strings.Replace(strings.Replace(sourceFile, filepath.Ext(sourceFile), "", 1), settings.SourceDirectory, "", 1) + for _, comparisonFile := range comparisonFiles { + comparisonFilePath := strings.Replace(strings.Replace(comparisonFile, filepath.Ext(comparisonFile), "", 1), settings.ComparisonDirectory, "", 1) + if comparisonFilePath != sourceFilePath { + continue + } + log.Debug("duplicate file found", sourceFile+" -> "+comparisonFile) + Duplicates = append(Duplicates, comparisonFile) + } } + log.InfoTimed("found "+strconv.Itoa(len(Duplicates))+" duplicate files", timestamp.Local().UnixMilli()) } // unexported function(s) diff --git a/go.mod b/go.mod index 1564760..c1ba55c 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module velvettear/dedupe -go 1.20 +go 1.21 diff --git a/main.go b/main.go index 9d753f6..5ff097c 100644 --- a/main.go +++ b/main.go @@ -1,11 +1,72 @@ package main import ( + "os" + "strconv" + "time" "velvettear/dedupe/files" + "velvettear/dedupe/log" + "velvettear/dedupe/prompts" "velvettear/dedupe/settings" ) func main() { + timestamp := time.Now() settings.Initialize() + log.Info("starting dedupe...") files.Scan() + if len(files.Duplicates) == 0 { + log.Info("no duplicate files have been found - exiting...") + os.Exit(0) + } + prompts.ListDuplicates() + var proceed bool + var deleteDuplicates bool + if len(settings.MoveDirectory) == 0 && !settings.Delete { + exit(timestamp, 0) + } + if len(settings.MoveDirectory) > 0 && settings.Delete { + proceed, deleteDuplicates = prompts.DeleteOrMove() + } else { + proceed = prompts.HandleDuplicates(settings.Delete) + } + if !proceed { + exit(timestamp, 0) + } + counter := 0 + subtimestamp := time.Now() + if deleteDuplicates { + log.Info("deleting duplicate files in '" + settings.ComparisonDirectory + "'...") + for _, file := range files.Duplicates { + subsubtimestamp := time.Now() + error := os.Remove(file) + if error != nil { + log.Error("encountered an error deleting the file '"+file+"'", error.Error()) + continue + } + counter++ + log.DebugTimed("deleted the file '"+file+"'", subsubtimestamp.UnixMilli()) + } + log.InfoTimed("deleted "+strconv.Itoa(counter)+" duplicate files to '"+settings.MoveDirectory+"'", subtimestamp.UnixMilli()) + } else { + log.Info("moving duplicate files to '" + settings.MoveDirectory + "'...") + for _, file := range files.Duplicates { + if files.MoveFile(file) { + counter++ + } + } + log.InfoTimed("moved "+strconv.Itoa(counter)+" duplicate files to '"+settings.MoveDirectory+"'", subtimestamp.UnixMilli()) + } + if prompts.DeleteEmptyDirectories(settings.SourceDirectory) { + files.RemoveEmptyDirectories(settings.SourceDirectory) + } + if prompts.DeleteEmptyDirectories(settings.ComparisonDirectory) { + files.RemoveEmptyDirectories(settings.ComparisonDirectory) + } + exit(timestamp, 0) +} + +func exit(timestamp time.Time, code int) { + log.InfoTimed("dedupe finished - exiting...", timestamp.UnixMilli()) + os.Exit(code) } diff --git a/prompts/prompts.go b/prompts/prompts.go new file mode 100644 index 0000000..bb63feb --- /dev/null +++ b/prompts/prompts.go @@ -0,0 +1,78 @@ +package prompts + +import ( + "bufio" + "os" + "strings" + "velvettear/dedupe/files" + "velvettear/dedupe/log" + "velvettear/dedupe/settings" +) + +func ListDuplicates() { + log.Info("would you like to list all duplicate files? [y]es | [n]o") + reader := bufio.NewReader(os.Stdin) + input, error := reader.ReadString('\n') + if error != nil { + log.Fatal("encountered an error reading the input", error.Error()) + } + input = strings.ToLower(strings.TrimSpace(strings.TrimSuffix(input, "\n"))) + if input != "y" && input != "yes" { + return + } + for _, duplicate := range files.Duplicates { + log.Info(duplicate) + } +} + +func HandleDuplicates(delete bool) bool { + var msg string + if delete { + msg = "are you sure you want to delete all duplicate files? [y]es | [n]o" + } else { + msg = "are you sure you want to move all duplicate files to '" + settings.MoveDirectory + "'? [y]es | [no]" + } + log.Info(msg) + reader := bufio.NewReader(os.Stdin) + input, error := reader.ReadString('\n') + if error != nil { + log.Fatal("encountered an error reading the input", error.Error()) + } + input = strings.ToLower(strings.TrimSpace(strings.TrimSuffix(input, "\n"))) + return input == "y" || input == "yes" +} + +func DeleteOrMove() (confirmed bool, delete bool) { + log.Warning("parameters for both actions 'delete' and 'move' are set") + log.Info("do you want to delete or move all duplicate files? [d]elete | [m]ove") + reader := bufio.NewReader(os.Stdin) + input, error := reader.ReadString('\n') + if error != nil { + log.Fatal("encountered an error reading the input", error.Error()) + } + input = strings.ToLower(strings.TrimSpace(strings.TrimSuffix(input, "\n"))) + switch input { + case "d": + fallthrough + case "delete": + return HandleDuplicates(true), true + case "m": + fallthrough + case "move": + return HandleDuplicates(false), false + default: + log.Fatal("input not recognized") + } + return false, false +} + +func DeleteEmptyDirectories(directory string) bool { + log.Info("do you want to delete all empty subdirectories in '" + directory + "'? [y]es | [n]o") + reader := bufio.NewReader(os.Stdin) + input, error := reader.ReadString('\n') + if error != nil { + log.Fatal("encountered an error reading the input", error.Error()) + } + input = strings.ToLower(strings.TrimSpace(strings.TrimSuffix(input, "\n"))) + return input == "y" || input == "yes" +} diff --git a/settings/arguments.go b/settings/arguments.go index 0257e57..a3db5fe 100644 --- a/settings/arguments.go +++ b/settings/arguments.go @@ -11,9 +11,17 @@ func Initialize() { if len(os.Args) < 3 { log.Fatal("error: missing arguments") } - for _, arg := range os.Args { + for index, arg := range os.Args { arg = strings.ToLower(arg) switch arg { + case "-m": + fallthrough + case "--move": + moveDirectoryIndex := index + 1 + if moveDirectoryIndex >= len(os.Args) { + log.Fatal("no move directory given") + } + setMoveDirectory(os.Args[index+1]) case "-d": fallthrough case "--delete": diff --git a/settings/variables.go b/settings/variables.go index 52f1757..e78bee6 100644 --- a/settings/variables.go +++ b/settings/variables.go @@ -1,6 +1,7 @@ package settings import ( + "os" "strconv" "velvettear/dedupe/log" ) @@ -10,6 +11,7 @@ var Verbose bool var Delete bool var SourceDirectory string var ComparisonDirectory string +var MoveDirectory string // unexported function(s) func setVerbose(verbose bool) { @@ -32,3 +34,15 @@ func setComparisonDirectory(directory string) { ComparisonDirectory = directory log.Debug("set source directory", ComparisonDirectory) } + +func setMoveDirectory(directory string) { + stats, error := os.Stat(directory) + if error != nil && os.IsNotExist(error) { + log.Fatal("given move directory '"+directory+"' does not exist", error.Error()) + } + if !stats.IsDir() { + log.Fatal("given move directory '" + directory + "' is not a valid directory") + } + MoveDirectory = directory + log.Debug("set move directory", MoveDirectory) +}