From a7ef9d8726da509c05d72bc0b90f94a4418c5b63 Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Sat, 8 Aug 2026 12:05:08 +0200 Subject: [PATCH] Buffer image hashing channel and feed candidates directly --- filesmap.go | 2 +- main.go | 55 ++++++++++++++++++++++++----------------------------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/filesmap.go b/filesmap.go index b800cb4..4166108 100644 --- a/filesmap.go +++ b/filesmap.go @@ -39,7 +39,7 @@ func newFilesMap() *FilesMap { fileSizes: map[string]int64{}, FilesHashing: make(chan *fileEntry), FilesHashed: make(chan *fileEntry, 100000), - ImagesHashing: make(chan *imageEntry), + ImagesHashing: make(chan *imageEntry, 100000), ImagesHashed: make(chan *imageEntry, 100000), progress: mpb.New(mpb.WithWidth(64)), } diff --git a/main.go b/main.go index 36c52b9..b685dfc 100644 --- a/main.go +++ b/main.go @@ -101,36 +101,8 @@ func main() { close(filesMap.FilesHashed) <-fileHashingDone - // Stage 3: image hashes are only needed for files with no file-hash - // collision: either they were never file-hashed (unique size) or their - // file hash did not match any other file. - var imageCandidates []*imageEntry - - // Files whose size is unique were never file-hashed, so they have no - // collision but may still be visually similar images. - for size, file := range filesMap.FilesBySize { - if file == "" { - continue - } - imageCandidates = append(imageCandidates, &imageEntry{path: file, size: size}) - } - - // Files that were file-hashed without colliding also need an image hash. - for _, files := range filesMap.FilesByHash { - if len(files) != 1 { - continue - } - imageCandidates = append(imageCandidates, &imageEntry{path: files[0], size: filesMap.fileSizes[files[0]]}) - } - - // Stage 4: compute the image hash of every candidate. - go func() { - for _, candidate := range imageCandidates { - filesMap.ImagesHashing <- candidate - } - close(filesMap.ImagesHashing) - }() - + // Start the image-hashing workers before feeding candidates, so the + // feed never blocks regardless of how many candidates there are. imageWG := sync.WaitGroup{} imageHashingDone := make(chan bool) for i := 0; i < runtime.GOMAXPROCS(0); i++ { @@ -139,6 +111,29 @@ func main() { } go filesMap.imageHashMergeWorker(imageHashingDone) + // Image hashes are only needed for files with no file-hash collision: + // either they were never file-hashed (unique size) or their file hash + // did not match any other file. Feed each candidate directly. + + // Files whose size is unique were never file-hashed, so they have no + // collision but may still be visually similar images. + for size, file := range filesMap.FilesBySize { + if file == "" { + continue + } + filesMap.ImagesHashing <- &imageEntry{path: file, size: size} + } + + // Files that were file-hashed without colliding also need an image hash. + for _, files := range filesMap.FilesByHash { + if len(files) != 1 { + continue + } + filesMap.ImagesHashing <- &imageEntry{path: files[0], size: filesMap.fileSizes[files[0]]} + } + + close(filesMap.ImagesHashing) + imageWG.Wait() close(filesMap.ImagesHashed) <-imageHashingDone