probably finished project
This commit is contained in:
parent
646732fa5f
commit
6b08d2807b
11 changed files with 289 additions and 10 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
@ -1,2 +1,2 @@
|
||||||
__debug_bin
|
__debug_bin*
|
||||||
worklog
|
dedupe
|
7
.vscode/launch.json
vendored
7
.vscode/launch.json
vendored
|
@ -8,10 +8,11 @@
|
||||||
"mode": "auto",
|
"mode": "auto",
|
||||||
"program": "${workspaceFolder}/main.go",
|
"program": "${workspaceFolder}/main.go",
|
||||||
"args": [
|
"args": [
|
||||||
"/tmp/nfs/music/lossless",
|
"/home/velvettear/music/lossless",
|
||||||
"/tmp/nfs/music/mp3",
|
"/home/velvettear/music/mp3",
|
||||||
"-v",
|
"-v",
|
||||||
"--delete"
|
"-m",
|
||||||
|
"/home/velvettear/music/duplicates"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
10
README.md
10
README.md
|
@ -1,3 +1,13 @@
|
||||||
# dedupe
|
# dedupe
|
||||||
|
|
||||||
simple command line tool to find and move/delete duplicate audio files
|
simple command line tool to find and move/delete duplicate audio files
|
||||||
|
|
||||||
|
## run
|
||||||
|
|
||||||
|
`dedupe [source] [comparison] (options)`
|
||||||
|
|
||||||
|
### options
|
||||||
|
|
||||||
|
- -d | --delete: delete duplicate files
|
||||||
|
- -m | --move: move duplicate files to specified directory
|
||||||
|
- -v | --verbose: enable verbose / debug output
|
||||||
|
|
55
files/cleaner.go
Normal file
55
files/cleaner.go
Normal file
|
@ -0,0 +1,55 @@
|
||||||
|
package files
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"velvettear/dedupe/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
var subdirectories []string
|
||||||
|
|
||||||
|
// exported function(s)
|
||||||
|
func RemoveEmptyDirectories(directory string) {
|
||||||
|
subdirectories = nil
|
||||||
|
timestamp := time.Now()
|
||||||
|
log.Info("deleting empty subdirectories...", "directory: "+directory)
|
||||||
|
filepath.WalkDir(directory, collectSubdirectories)
|
||||||
|
sort.Slice(subdirectories, func(i, j int) bool {
|
||||||
|
return strings.Count(subdirectories[i], string(os.PathSeparator)) > strings.Count(subdirectories[j], string(os.PathSeparator))
|
||||||
|
})
|
||||||
|
count := 0
|
||||||
|
for _, subdirectory := range subdirectories {
|
||||||
|
files, error := os.ReadDir(subdirectory)
|
||||||
|
if error != nil {
|
||||||
|
log.Error("encountered an error checking the content of directory '" + subdirectory)
|
||||||
|
}
|
||||||
|
if len(files) > 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
error = os.Remove(subdirectory)
|
||||||
|
if error != nil {
|
||||||
|
log.Warning("encountered an error deleting directory '"+subdirectory+"'", error.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
count++
|
||||||
|
log.Debug("deleted directory '" + subdirectory + "'")
|
||||||
|
}
|
||||||
|
log.InfoTimed("finished deleting "+strconv.Itoa(count)+" empty subdirectoies", timestamp.UnixMilli(), "directory: "+directory)
|
||||||
|
}
|
||||||
|
|
||||||
|
// unexported function(s)
|
||||||
|
func collectSubdirectories(path string, dir fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !dir.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
subdirectories = append(subdirectories, path)
|
||||||
|
return nil
|
||||||
|
}
|
39
files/mover.go
Normal file
39
files/mover.go
Normal file
|
@ -0,0 +1,39 @@
|
||||||
|
package files
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"velvettear/dedupe/log"
|
||||||
|
"velvettear/dedupe/settings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// exported function(s)
|
||||||
|
func MoveFile(file string) bool {
|
||||||
|
timestamp := time.Now()
|
||||||
|
targetFile := filepath.Join(settings.MoveDirectory, strings.Replace(file, settings.ComparisonDirectory, "", 1))
|
||||||
|
targetDirectory := filepath.Dir(targetFile)
|
||||||
|
error := createTargetDirectory(targetDirectory, 0777)
|
||||||
|
if error != nil {
|
||||||
|
log.Error("encountered an error creating the directory '"+targetDirectory+"'", error.Error())
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
error = os.Rename(file, targetFile)
|
||||||
|
if error != nil {
|
||||||
|
log.Error("encountered an error moving the file '"+file+"' to '"+targetFile+"'", error.Error())
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
log.DebugTimed("moved file '"+file+"' to '"+targetFile+"'", timestamp.UnixMilli())
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func createTargetDirectory(directory string, permissions fs.FileMode) error {
|
||||||
|
error := os.MkdirAll(directory, permissions)
|
||||||
|
if error != nil {
|
||||||
|
return error
|
||||||
|
}
|
||||||
|
log.Debug("created directory '" + directory + "'")
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -4,11 +4,14 @@ import (
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"velvettear/dedupe/log"
|
"velvettear/dedupe/log"
|
||||||
"velvettear/dedupe/settings"
|
"velvettear/dedupe/settings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var Duplicates []string
|
||||||
|
|
||||||
var sourceFiles []string
|
var sourceFiles []string
|
||||||
var comparisonFiles []string
|
var comparisonFiles []string
|
||||||
|
|
||||||
|
@ -24,11 +27,21 @@ func Scan() {
|
||||||
filepath.WalkDir(settings.ComparisonDirectory, fillComparisonFiles)
|
filepath.WalkDir(settings.ComparisonDirectory, fillComparisonFiles)
|
||||||
log.InfoTimed("found "+strconv.Itoa(len(comparisonFiles))+" comparison files", timestamp.UnixMilli())
|
log.InfoTimed("found "+strconv.Itoa(len(comparisonFiles))+" comparison files", timestamp.UnixMilli())
|
||||||
|
|
||||||
|
timestamp = time.Now()
|
||||||
|
log.Info("comparing files...")
|
||||||
for _, sourceFile := range sourceFiles {
|
for _, sourceFile := range sourceFiles {
|
||||||
log.Debug("checking file", sourceFile)
|
log.Debug("checking file", sourceFile)
|
||||||
sourceFileName := filepath.Base(sourceFile)
|
sourceFilePath := strings.Replace(strings.Replace(sourceFile, filepath.Ext(sourceFile), "", 1), settings.SourceDirectory, "", 1)
|
||||||
log.Debug("derp", sourceFileName)
|
for _, comparisonFile := range comparisonFiles {
|
||||||
|
comparisonFilePath := strings.Replace(strings.Replace(comparisonFile, filepath.Ext(comparisonFile), "", 1), settings.ComparisonDirectory, "", 1)
|
||||||
|
if comparisonFilePath != sourceFilePath {
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
log.Debug("duplicate file found", sourceFile+" -> "+comparisonFile)
|
||||||
|
Duplicates = append(Duplicates, comparisonFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.InfoTimed("found "+strconv.Itoa(len(Duplicates))+" duplicate files", timestamp.Local().UnixMilli())
|
||||||
}
|
}
|
||||||
|
|
||||||
// unexported function(s)
|
// unexported function(s)
|
2
go.mod
2
go.mod
|
@ -1,3 +1,3 @@
|
||||||
module velvettear/dedupe
|
module velvettear/dedupe
|
||||||
|
|
||||||
go 1.20
|
go 1.21
|
||||||
|
|
61
main.go
61
main.go
|
@ -1,11 +1,72 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
"velvettear/dedupe/files"
|
"velvettear/dedupe/files"
|
||||||
|
"velvettear/dedupe/log"
|
||||||
|
"velvettear/dedupe/prompts"
|
||||||
"velvettear/dedupe/settings"
|
"velvettear/dedupe/settings"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
timestamp := time.Now()
|
||||||
settings.Initialize()
|
settings.Initialize()
|
||||||
|
log.Info("starting dedupe...")
|
||||||
files.Scan()
|
files.Scan()
|
||||||
|
if len(files.Duplicates) == 0 {
|
||||||
|
log.Info("no duplicate files have been found - exiting...")
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
prompts.ListDuplicates()
|
||||||
|
var proceed bool
|
||||||
|
var deleteDuplicates bool
|
||||||
|
if len(settings.MoveDirectory) == 0 && !settings.Delete {
|
||||||
|
exit(timestamp, 0)
|
||||||
|
}
|
||||||
|
if len(settings.MoveDirectory) > 0 && settings.Delete {
|
||||||
|
proceed, deleteDuplicates = prompts.DeleteOrMove()
|
||||||
|
} else {
|
||||||
|
proceed = prompts.HandleDuplicates(settings.Delete)
|
||||||
|
}
|
||||||
|
if !proceed {
|
||||||
|
exit(timestamp, 0)
|
||||||
|
}
|
||||||
|
counter := 0
|
||||||
|
subtimestamp := time.Now()
|
||||||
|
if deleteDuplicates {
|
||||||
|
log.Info("deleting duplicate files in '" + settings.ComparisonDirectory + "'...")
|
||||||
|
for _, file := range files.Duplicates {
|
||||||
|
subsubtimestamp := time.Now()
|
||||||
|
error := os.Remove(file)
|
||||||
|
if error != nil {
|
||||||
|
log.Error("encountered an error deleting the file '"+file+"'", error.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
counter++
|
||||||
|
log.DebugTimed("deleted the file '"+file+"'", subsubtimestamp.UnixMilli())
|
||||||
|
}
|
||||||
|
log.InfoTimed("deleted "+strconv.Itoa(counter)+" duplicate files to '"+settings.MoveDirectory+"'", subtimestamp.UnixMilli())
|
||||||
|
} else {
|
||||||
|
log.Info("moving duplicate files to '" + settings.MoveDirectory + "'...")
|
||||||
|
for _, file := range files.Duplicates {
|
||||||
|
if files.MoveFile(file) {
|
||||||
|
counter++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.InfoTimed("moved "+strconv.Itoa(counter)+" duplicate files to '"+settings.MoveDirectory+"'", subtimestamp.UnixMilli())
|
||||||
|
}
|
||||||
|
if prompts.DeleteEmptyDirectories(settings.SourceDirectory) {
|
||||||
|
files.RemoveEmptyDirectories(settings.SourceDirectory)
|
||||||
|
}
|
||||||
|
if prompts.DeleteEmptyDirectories(settings.ComparisonDirectory) {
|
||||||
|
files.RemoveEmptyDirectories(settings.ComparisonDirectory)
|
||||||
|
}
|
||||||
|
exit(timestamp, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func exit(timestamp time.Time, code int) {
|
||||||
|
log.InfoTimed("dedupe finished - exiting...", timestamp.UnixMilli())
|
||||||
|
os.Exit(code)
|
||||||
}
|
}
|
||||||
|
|
78
prompts/prompts.go
Normal file
78
prompts/prompts.go
Normal file
|
@ -0,0 +1,78 @@
|
||||||
|
package prompts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"velvettear/dedupe/files"
|
||||||
|
"velvettear/dedupe/log"
|
||||||
|
"velvettear/dedupe/settings"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ListDuplicates() {
|
||||||
|
log.Info("would you like to list all duplicate files? [y]es | [n]o")
|
||||||
|
reader := bufio.NewReader(os.Stdin)
|
||||||
|
input, error := reader.ReadString('\n')
|
||||||
|
if error != nil {
|
||||||
|
log.Fatal("encountered an error reading the input", error.Error())
|
||||||
|
}
|
||||||
|
input = strings.ToLower(strings.TrimSpace(strings.TrimSuffix(input, "\n")))
|
||||||
|
if input != "y" && input != "yes" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, duplicate := range files.Duplicates {
|
||||||
|
log.Info(duplicate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func HandleDuplicates(delete bool) bool {
|
||||||
|
var msg string
|
||||||
|
if delete {
|
||||||
|
msg = "are you sure you want to delete all duplicate files? [y]es | [n]o"
|
||||||
|
} else {
|
||||||
|
msg = "are you sure you want to move all duplicate files to '" + settings.MoveDirectory + "'? [y]es | [no]"
|
||||||
|
}
|
||||||
|
log.Info(msg)
|
||||||
|
reader := bufio.NewReader(os.Stdin)
|
||||||
|
input, error := reader.ReadString('\n')
|
||||||
|
if error != nil {
|
||||||
|
log.Fatal("encountered an error reading the input", error.Error())
|
||||||
|
}
|
||||||
|
input = strings.ToLower(strings.TrimSpace(strings.TrimSuffix(input, "\n")))
|
||||||
|
return input == "y" || input == "yes"
|
||||||
|
}
|
||||||
|
|
||||||
|
func DeleteOrMove() (confirmed bool, delete bool) {
|
||||||
|
log.Warning("parameters for both actions 'delete' and 'move' are set")
|
||||||
|
log.Info("do you want to delete or move all duplicate files? [d]elete | [m]ove")
|
||||||
|
reader := bufio.NewReader(os.Stdin)
|
||||||
|
input, error := reader.ReadString('\n')
|
||||||
|
if error != nil {
|
||||||
|
log.Fatal("encountered an error reading the input", error.Error())
|
||||||
|
}
|
||||||
|
input = strings.ToLower(strings.TrimSpace(strings.TrimSuffix(input, "\n")))
|
||||||
|
switch input {
|
||||||
|
case "d":
|
||||||
|
fallthrough
|
||||||
|
case "delete":
|
||||||
|
return HandleDuplicates(true), true
|
||||||
|
case "m":
|
||||||
|
fallthrough
|
||||||
|
case "move":
|
||||||
|
return HandleDuplicates(false), false
|
||||||
|
default:
|
||||||
|
log.Fatal("input not recognized")
|
||||||
|
}
|
||||||
|
return false, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func DeleteEmptyDirectories(directory string) bool {
|
||||||
|
log.Info("do you want to delete all empty subdirectories in '" + directory + "'? [y]es | [n]o")
|
||||||
|
reader := bufio.NewReader(os.Stdin)
|
||||||
|
input, error := reader.ReadString('\n')
|
||||||
|
if error != nil {
|
||||||
|
log.Fatal("encountered an error reading the input", error.Error())
|
||||||
|
}
|
||||||
|
input = strings.ToLower(strings.TrimSpace(strings.TrimSuffix(input, "\n")))
|
||||||
|
return input == "y" || input == "yes"
|
||||||
|
}
|
|
@ -11,9 +11,17 @@ func Initialize() {
|
||||||
if len(os.Args) < 3 {
|
if len(os.Args) < 3 {
|
||||||
log.Fatal("error: missing arguments")
|
log.Fatal("error: missing arguments")
|
||||||
}
|
}
|
||||||
for _, arg := range os.Args {
|
for index, arg := range os.Args {
|
||||||
arg = strings.ToLower(arg)
|
arg = strings.ToLower(arg)
|
||||||
switch arg {
|
switch arg {
|
||||||
|
case "-m":
|
||||||
|
fallthrough
|
||||||
|
case "--move":
|
||||||
|
moveDirectoryIndex := index + 1
|
||||||
|
if moveDirectoryIndex >= len(os.Args) {
|
||||||
|
log.Fatal("no move directory given")
|
||||||
|
}
|
||||||
|
setMoveDirectory(os.Args[index+1])
|
||||||
case "-d":
|
case "-d":
|
||||||
fallthrough
|
fallthrough
|
||||||
case "--delete":
|
case "--delete":
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package settings
|
package settings
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
"velvettear/dedupe/log"
|
"velvettear/dedupe/log"
|
||||||
)
|
)
|
||||||
|
@ -10,6 +11,7 @@ var Verbose bool
|
||||||
var Delete bool
|
var Delete bool
|
||||||
var SourceDirectory string
|
var SourceDirectory string
|
||||||
var ComparisonDirectory string
|
var ComparisonDirectory string
|
||||||
|
var MoveDirectory string
|
||||||
|
|
||||||
// unexported function(s)
|
// unexported function(s)
|
||||||
func setVerbose(verbose bool) {
|
func setVerbose(verbose bool) {
|
||||||
|
@ -32,3 +34,15 @@ func setComparisonDirectory(directory string) {
|
||||||
ComparisonDirectory = directory
|
ComparisonDirectory = directory
|
||||||
log.Debug("set source directory", ComparisonDirectory)
|
log.Debug("set source directory", ComparisonDirectory)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func setMoveDirectory(directory string) {
|
||||||
|
stats, error := os.Stat(directory)
|
||||||
|
if error != nil && os.IsNotExist(error) {
|
||||||
|
log.Fatal("given move directory '"+directory+"' does not exist", error.Error())
|
||||||
|
}
|
||||||
|
if !stats.IsDir() {
|
||||||
|
log.Fatal("given move directory '" + directory + "' is not a valid directory")
|
||||||
|
}
|
||||||
|
MoveDirectory = directory
|
||||||
|
log.Debug("set move directory", MoveDirectory)
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue