6 Commits
Author SHA1 Message Date
Jan Bader e40c3c8560 do not pass erred files on 2026-04-13 21:48:07 +02:00
Jan Bader 683ebf0858 use pointer for entries
hash was not passed on from FileHashingWorker
2026-04-13 21:47:50 +02:00
Jan Bader 3c6ae055a6 Add flake for devshell 2025-07-09 22:46:02 +02:00
Jan Bader 7c95ec57d7 Ignore tmp 2024-10-04 14:33:20 +02:00
Jan Bader c72fbdd086 Improve display for progress 2024-10-04 14:32:59 +02:00
Jan Bader ce574a0b4a Ignore test_data 2024-10-04 14:32:45 +02:00
5 changed files with 105 additions and 17 deletions

No files matched your search

+1
View File
@@ -0,0 +1 @@
use flake
+3
View File
@@ -1 +1,4 @@
*.exe
test_data/
tmp/
.direnv/
+16 -17
View File
@@ -14,13 +14,13 @@ import (
// FilesMap is a struct for listing files by Size and Hash to search for duplicates
type FilesMap struct {
Images []imageEntry
Images []*imageEntry
FilesBySize map[int64]string
FilesByHash map[string][]string
FilesHashing chan fileEntry
FilesHashed chan fileEntry
ImagesHashing chan imageEntry
ImagesHashed chan imageEntry
FilesHashing chan *fileEntry
FilesHashed chan *fileEntry
ImagesHashing chan *imageEntry
ImagesHashed chan *imageEntry
progress *mpb.Progress
incomingBar *mpb.Bar
fileHashingBar *mpb.Bar
@@ -32,10 +32,10 @@ func newFilesMap() *FilesMap {
return &FilesMap{
FilesBySize: map[int64]string{},
FilesByHash: map[string][]string{},
FilesHashed: make(chan fileEntry, 100000),
FilesHashing: make(chan fileEntry),
ImagesHashed: make(chan imageEntry, 100000),
ImagesHashing: make(chan imageEntry),
FilesHashed: make(chan *fileEntry, 100000),
FilesHashing: make(chan *fileEntry),
ImagesHashed: make(chan *imageEntry, 100000),
ImagesHashing: make(chan *imageEntry),
progress: mpb.New(mpb.WithWidth(64)),
}
}
@@ -43,19 +43,18 @@ func newFilesMap() *FilesMap {
func (fm *FilesMap) FileHashingWorker(wg *sync.WaitGroup) {
for file := range fm.FilesHashing {
if *verbose {
fmt.Println("Hashing file", file.path)
fmt.Fprintf(fm.progress, "Hashing file %s\n", file.path)
}
hash, err := calculateFileHash(file.path)
fm.fileHashingBar.IncrInt64(file.size)
fm.FilesHashed <- file
if err != nil {
fmt.Fprintf(fm.progress, "Error calculating Hash for file %s: %v\n", file.path, err)
continue
}
file.hash = hash
fm.FilesHashed <- file
}
wg.Done()
}
@@ -63,7 +62,7 @@ func (fm *FilesMap) FileHashingWorker(wg *sync.WaitGroup) {
func (fm *FilesMap) ImageHashingWorker(wg *sync.WaitGroup) {
for file := range fm.ImagesHashing {
if *verbose {
fmt.Println("Hashing image", file.path)
fmt.Fprintf(fm.progress, "Hashing image %s\n", file.path)
}
hash, err := calculateImageHash(file.path)
@@ -85,7 +84,7 @@ func (fm *FilesMap) ImageHashingWorker(wg *sync.WaitGroup) {
func (fm *FilesMap) HashedWorker(done chan bool) {
for file := range fm.FilesHashed {
if *verbose {
fmt.Println("Finishing", file.path)
fmt.Printf("Finishing '%s' with hash '%s'\n", file.path, file.hash)
}
fm.lock.Lock()
@@ -152,9 +151,9 @@ func (fm *FilesMap) hashFile(path string, size int64) int64 {
fmt.Println("Incoming", path)
}
fm.FilesHashing <- fileEntry{path, size, ""}
fm.FilesHashing <- &fileEntry{path, size, ""}
if prevFile != "" {
fm.FilesHashing <- fileEntry{prevFile, size, ""}
fm.FilesHashing <- &fileEntry{prevFile, size, ""}
return 2
}
@@ -162,7 +161,7 @@ func (fm *FilesMap) hashFile(path string, size int64) int64 {
}
func (fm *FilesMap) hashImage(path string, size int64) {
fm.ImagesHashing <- imageEntry{path, size, 0}
fm.ImagesHashing <- &imageEntry{path, size, 0}
}
type imageEntry struct {
Generated
+60
View File
@@ -0,0 +1,60 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1751984180,
"narHash": "sha256-LwWRsENAZJKUdD3SpLluwDmdXY9F45ZEgCb0X+xgOL0=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "9807714d6944a957c2e036f84b0ff8caf9930bc0",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-unstable",
"type": "indirect"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs",
"utils": "utils"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
}
},
"root": "root",
"version": 7
}
+25
View File
@@ -0,0 +1,25 @@
{
inputs = {
utils.url = "github:numtide/flake-utils";
nixpkgs.url = "nixpkgs/nixos-unstable";
};
outputs = { self, nixpkgs, utils }:
utils.lib.eachDefaultSystem (system:
let
pkgs = nixpkgs.legacyPackages."${system}";
in rec {
devShell = pkgs.mkShell {
shellHook = ''
export SHELL=${pkgs.fish}/bin/fish
'';
nativeBuildInputs = with pkgs; [
go_1_24
gopls
golangci-lint
golangci-lint-langserver
];
};
}
);
}