commit 4980f16e0dbf7c28c395f1a785f693bdf09d0f1b
parent 8116c3476308e643eb98357611f689236e7c0b56
Author: Peter <peter@minskio.co.uk>
Date: Sat, 24 Oct 2026 11:44:54 +0100
Add preliminary czkawka duplicate finding
Diffstat:
7 files changed, 886 insertions(+), 3 deletions(-)
diff --git a/internal/app/admin_duplicates.go b/internal/app/admin_duplicates.go
@@ -0,0 +1,586 @@
+package app
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "sync"
+)
+
+// Binary name
+var CzkawkaBin = "czkawka_cli"
+// dupEntryRe matches lines like: "/path/to/img.jpg" - 1280x1280 - 124.57 KiB - Original
+var dupEntryRe = regexp.MustCompile(`^"(.*)" - (\d+x\d+) - (.+?) - ([^-]+)$`)
+
+type DupItem struct {
+ Path string `json:"path"`
+ Dims string `json:"dims"`
+ Size string `json:"size"`
+ Similarity string `json:"similarity"`
+}
+
+type dupState struct {
+ mu sync.Mutex
+ dirs []string
+ groups [][]DupItem
+ allowed map[string]bool
+}
+var dups = &dupState{allowed: map[string]bool{}}
+
+// Only allow one czkawka scan at a time.
+var scanMu sync.Mutex
+
+// read report text output and parse into groups
+func parseDuplicateResults(file string) ([][]DupItem, error) {
+ data, err := os.ReadFile(file)
+ if err != nil {
+ return nil, err
+ }
+
+ var groups [][]DupItem
+ var cur []DupItem
+ flush := func() {
+ if len(cur) >= 2 {
+ groups = append(groups, cur)
+ }
+ cur = nil
+ }
+
+ for _, line := range strings.Split(string(data), "\n") {
+ line = strings.TrimRight(line, "\r")
+ if m := dupEntryRe.FindStringSubmatch(line); m != nil {
+ cur = append(cur, DupItem{
+ Path: m[1],
+ Dims: m[2],
+ Size: m[3],
+ Similarity: strings.TrimSpace(m[4]),
+ })
+ continue
+ }
+ // Anything else (blank line, "Found N images...", headers) ends a group.
+ flush()
+ }
+ flush()
+ return groups, nil
+}
+
+func (s *dupState) set(dirs []string, groups [][]DupItem) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.dirs = dirs
+ s.groups = groups
+ s.allowed = map[string]bool{}
+ for _, g := range groups {
+ for _, it := range g {
+ s.allowed[it.Path] = true
+ }
+ }
+}
+
+// liveGroups drops groups missing files and with less than 2 files
+func (s *dupState) liveGroups() [][]DupItem {
+ out := make([][]DupItem, 0, len(s.groups))
+ for _, g := range s.groups {
+ var live []DupItem
+ for _, it := range g {
+ if _, err := os.Stat(it.Path); err == nil {
+ live = append(live, it)
+ }
+ }
+ if len(live) >= 2 {
+ out = append(out, live)
+ }
+ }
+ return out
+}
+
+// sameGroup reports whether a and b appear together in one results group.
+func (s *dupState) sameGroup(a, b string) bool {
+ for _, g := range s.groups {
+ var hasA, hasB bool
+ for _, it := range g {
+ hasA = hasA || it.Path == a
+ hasB = hasB || it.Path == b
+ }
+ if hasA && hasB {
+ return true
+ }
+ }
+ return false
+}
+
+// remainingInGroup counts the existing files in the group containing path.
+func (s *dupState) remainingInGroup(path string) int {
+ for _, g := range s.groups {
+ found := false
+ for _, it := range g {
+ if it.Path == path {
+ found = true
+ break
+ }
+ }
+ if !found {
+ continue
+ }
+ n := 0
+ for _, it := range g {
+ if _, err := os.Stat(it.Path); err == nil {
+ n++
+ }
+ }
+ return n
+ }
+ return 0
+}
+
+// dupResponse is what the scan and groups endpoints return.
+type dupResponse struct {
+ Dirs []string `json:"dirs"`
+ Groups [][]DupItem `json:"groups"`
+}
+
+func writeJSON(w http.ResponseWriter, v any) {
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(v); err != nil {
+ log.Printf("Warning: writeJSON: %v", err)
+ }
+}
+
+// check JSON POST for CORS so other sites can't request actions
+func decodeJSONPost(w http.ResponseWriter, r *http.Request, v any) bool {
+ if r.Method != http.MethodPost {
+ http.Error(w, "POST only", http.StatusMethodNotAllowed)
+ return false
+ }
+ if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
+ http.Error(w, "expected application/json", http.StatusUnsupportedMediaType)
+ return false
+ }
+ if err := json.NewDecoder(r.Body).Decode(v); err != nil {
+ http.Error(w, "bad request", http.StatusBadRequest)
+ return false
+ }
+ return true
+}
+
+// cleans submitted directories, checks they exist and drops blanks and duplicates
+func resolveScanDirs(inputs []string) ([]string, error) {
+ var dirs []string
+ seen := map[string]bool{}
+ for _, in := range inputs {
+ in = strings.TrimSpace(in)
+ if in == "" {
+ continue
+ }
+ abs, err := filepath.Abs(in)
+ if err != nil {
+ return nil, fmt.Errorf("invalid path %q: %w", in, err)
+ }
+ info, err := os.Stat(abs)
+ if err != nil {
+ return nil, fmt.Errorf("cannot read %s: %w", abs, err)
+ }
+ if !info.IsDir() {
+ return nil, fmt.Errorf("%s is not a directory", abs)
+ }
+ if !seen[abs] {
+ seen[abs] = true
+ dirs = append(dirs, abs)
+ }
+ }
+ if len(dirs) == 0 {
+ return nil, errors.New("enter at least one directory to scan")
+ }
+ return dirs, nil
+}
+
+// pathWithin reports whether p is dir itself or located beneath it.
+func pathWithin(p, dir string) bool {
+ rel, err := filepath.Rel(dir, p)
+ if err != nil {
+ return false
+ }
+ return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)))
+}
+
+// run binary and write report
+func runCzkawka(ctx context.Context, resultsFile string, dirs []string) error {
+ args := []string{"image"}
+ for _, d := range dirs {
+ args = append(args, "-d", d)
+ }
+ // Video thumbnails live in the uploads directory; they are derived files, not duplicates.
+ if uploadAbs, err := filepath.Abs(Cfg.UploadDir); err == nil {
+ thumbDir := filepath.Join(uploadAbs, "thumbnails")
+ for _, d := range dirs {
+ if pathWithin(thumbDir, d) {
+ if info, err := os.Stat(thumbDir); err == nil && info.IsDir() {
+ args = append(args, "-e", thumbDir)
+ }
+ break
+ }
+ }
+ }
+ args = append(args, "-f", resultsFile)
+
+ log.Printf("Info: runCzkawka: %s %s", CzkawkaBin, strings.Join(args, " "))
+ var out bytes.Buffer
+ cmd := exec.CommandContext(ctx, CzkawkaBin, args...)
+ cmd.Stdout = &out
+ cmd.Stderr = &out
+ err := cmd.Run()
+
+ // czkawka_cli exits with 11 when it found results; that's not a failure.
+ var ee *exec.ExitError
+ if errors.As(err, &ee) && ee.ExitCode() == 11 {
+ return nil
+ }
+ if err != nil {
+ if errors.Is(err, exec.ErrNotFound) {
+ return fmt.Errorf("%s was not found; install czkawka_cli or pass its location with the -czkawka flag", CzkawkaBin)
+ }
+ tail := strings.TrimSpace(out.String())
+ if len(tail) > 600 {
+ tail = "..." + tail[len(tail)-600:]
+ }
+ if tail != "" {
+ return fmt.Errorf("czkawka failed: %w: %s", err, tail)
+ }
+ return fmt.Errorf("czkawka failed: %w", err)
+ }
+ return nil
+}
+
+func duplicatesScanHandler(w http.ResponseWriter, r *http.Request) {
+ var req struct {
+ Dirs []string `json:"dirs"`
+ }
+ if !decodeJSONPost(w, r, &req) {
+ return
+ }
+
+ dirs, err := resolveScanDirs(req.Dirs)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ if !scanMu.TryLock() {
+ http.Error(w, "a scan is already running", http.StatusConflict)
+ return
+ }
+ defer scanMu.Unlock()
+
+ tmpDir, err := os.MkdirTemp("", "tagliatelle-duplicates-")
+ if err != nil {
+ log.Printf("Error: duplicatesScanHandler: failed to create temp dir: %v", err)
+ http.Error(w, "failed to create temporary directory", http.StatusInternalServerError)
+ return
+ }
+ defer os.RemoveAll(tmpDir)
+ resultsFile := filepath.Join(tmpDir, "duplicates.log")
+
+ if err := runCzkawka(r.Context(), resultsFile, dirs); err != nil {
+ log.Printf("Error: duplicatesScanHandler: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ groups, err := parseDuplicateResults(resultsFile)
+ if errors.Is(err, os.ErrNotExist) {
+ groups, err = nil, nil // czkawka writes no file when nothing was found
+ }
+ if err != nil {
+ log.Printf("Error: duplicatesScanHandler: failed to read results: %v", err)
+ http.Error(w, "failed to read czkawka results", http.StatusInternalServerError)
+ return
+ }
+ log.Printf("Info: duplicatesScanHandler: found %d groups in %s", len(groups), strings.Join(dirs, ", "))
+
+ dups.set(dirs, groups)
+ dups.mu.Lock()
+ defer dups.mu.Unlock()
+ writeJSON(w, dupResponse{Dirs: dirs, Groups: dups.liveGroups()})
+}
+
+func duplicatesGroupsHandler(w http.ResponseWriter, r *http.Request) {
+ dups.mu.Lock()
+ defer dups.mu.Unlock()
+ dirs := dups.dirs
+ if dirs == nil {
+ dirs = []string{}
+ }
+ writeJSON(w, dupResponse{Dirs: dirs, Groups: dups.liveGroups()})
+}
+
+func duplicatesImageHandler(w http.ResponseWriter, r *http.Request) {
+ p := r.URL.Query().Get("path")
+ dups.mu.Lock()
+ ok := dups.allowed[p]
+ dups.mu.Unlock()
+ if !ok {
+ http.Error(w, "not found", http.StatusNotFound)
+ return
+ }
+ w.Header().Set("Cache-Control", "private, max-age=300")
+ http.ServeFile(w, r, p)
+}
+
+func duplicatesDeleteHandler(w http.ResponseWriter, r *http.Request) {
+ var req struct {
+ Path string `json:"path"`
+ }
+ if !decodeJSONPost(w, r, &req) {
+ return
+ }
+
+ dups.mu.Lock()
+ defer dups.mu.Unlock()
+ if !dups.allowed[req.Path] {
+ http.Error(w, "path not in results", http.StatusForbidden)
+ return
+ }
+ if dups.remainingInGroup(req.Path) < 2 {
+ http.Error(w, "refusing to delete the last remaining file in a group", http.StatusConflict)
+ return
+ }
+ if err := os.Remove(req.Path); err != nil && !errors.Is(err, os.ErrNotExist) {
+ log.Printf("Error: duplicatesDeleteHandler: failed to delete %s: %v", req.Path, err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ forgetUploadRecord(req.Path)
+ log.Printf("Info: duplicatesDeleteHandler: deleted %s", req.Path)
+ w.WriteHeader(http.StatusNoContent)
+}
+
+// duplicatesDeleteGroupHandler deletes every listed file of one results group,
+func duplicatesDeleteGroupHandler(w http.ResponseWriter, r *http.Request) {
+ var req struct {
+ Paths []string `json:"paths"`
+ }
+ if !decodeJSONPost(w, r, &req) {
+ return
+ }
+ if len(req.Paths) == 0 {
+ http.Error(w, "no paths given", http.StatusBadRequest)
+ return
+ }
+
+ dups.mu.Lock()
+ defer dups.mu.Unlock()
+ for _, p := range req.Paths {
+ if !dups.allowed[p] || !dups.sameGroup(req.Paths[0], p) {
+ http.Error(w, "paths are not in one group of the results", http.StatusForbidden)
+ return
+ }
+ }
+ var failed []string
+ for _, p := range req.Paths {
+ if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) {
+ log.Printf("Error: duplicatesDeleteGroupHandler: failed to delete %s: %v", p, err)
+ failed = append(failed, p)
+ continue
+ }
+ forgetUploadRecord(p)
+ log.Printf("Info: duplicatesDeleteGroupHandler: deleted %s", p)
+ }
+ if len(failed) > 0 {
+ http.Error(w, "could not delete: "+strings.Join(failed, ", "), http.StatusInternalServerError)
+ return
+ }
+ w.WriteHeader(http.StatusNoContent)
+}
+
+func duplicatesReplaceHandler(w http.ResponseWriter, r *http.Request) {
+ var req struct {
+ Src string `json:"src"` // the better file; it is moved
+ Dst string `json:"dst"` // the file being replaced
+ }
+ if !decodeJSONPost(w, r, &req) {
+ return
+ }
+
+ dups.mu.Lock()
+ defer dups.mu.Unlock()
+ if req.Src == req.Dst {
+ http.Error(w, "src and dst are the same file", http.StatusBadRequest)
+ return
+ }
+ if !dups.allowed[req.Src] || !dups.allowed[req.Dst] || !dups.sameGroup(req.Src, req.Dst) {
+ http.Error(w, "paths are not a matching pair in the results", http.StatusForbidden)
+ return
+ }
+
+ dest, err := replaceFile(req.Src, req.Dst)
+ if err != nil {
+ log.Printf("Error: duplicatesReplaceHandler: %s -> %s: %v", req.Src, req.Dst, err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ syncUploadRecordsAfterReplace(req.Src, req.Dst, dest)
+ log.Printf("Info: duplicatesReplaceHandler: replaced %s with %s (now at %s)", req.Dst, req.Src, dest)
+ writeJSON(w, map[string]string{"path": dest})
+}
+
+// moves src into dst's location
+// On extension mismatch, keep src's extension and remove dst's
+func replaceFile(src, dst string) (string, error) {
+ dstInfo, err := os.Stat(dst)
+ if err != nil {
+ return "", err
+ }
+ if _, err := os.Stat(src); err != nil {
+ return "", err
+ }
+
+ dest := dst
+ if sx, dx := filepath.Ext(src), filepath.Ext(dst); !strings.EqualFold(sx, dx) {
+ dest = strings.TrimSuffix(dst, dx) + sx
+ if _, err := os.Lstat(dest); err == nil {
+ return "", fmt.Errorf("%s already exists", dest)
+ }
+ }
+
+ if err := moveFile(src, dest); err != nil {
+ return "", err
+ }
+ // try to keep existing permissions
+ _ = os.Chmod(dest, dstInfo.Mode().Perm())
+
+ if dest != dst {
+ if err := os.Remove(dst); err != nil {
+ return dest, fmt.Errorf("moved to %s, but could not remove old file: %w", dest, err)
+ }
+ }
+ return dest, nil
+}
+
+// rename src to dest. If unable, copy to temp and rename in place then remove src
+func moveFile(src, dest string) error {
+ if err := os.Rename(src, dest); err == nil {
+ return nil
+ }
+
+ in, err := os.Open(src)
+ if err != nil {
+ return err
+ }
+ defer in.Close()
+
+ tmp, err := os.CreateTemp(filepath.Dir(dest), ".replace-*")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(tmp.Name()) // no-op once renamed
+
+ if _, err := io.Copy(tmp, in); err != nil {
+ tmp.Close()
+ return err
+ }
+ if err := tmp.Sync(); err != nil {
+ tmp.Close()
+ return err
+ }
+ if err := tmp.Close(); err != nil {
+ return err
+ }
+ if err := os.Rename(tmp.Name(), dest); err != nil {
+ return err
+ }
+ if err := os.Remove(src); err != nil {
+ return fmt.Errorf("copied to %s, but could not remove source: %w", dest, err)
+ }
+ return nil
+}
+
+// try to keep the tagliatelle database in sync
+func uploadRecord(absPath string) (id int64, filename, relPath string, ok bool) {
+ uploadAbs, err := filepath.Abs(Cfg.UploadDir)
+ if err != nil || !pathWithin(absPath, uploadAbs) {
+ return 0, "", "", false
+ }
+ rel, err := filepath.Rel(uploadAbs, absPath)
+ if err != nil {
+ return 0, "", "", false
+ }
+ if err := DB.QueryRow("SELECT id, filename FROM files WHERE path = ?", rel).Scan(&id, &filename); err != nil {
+ return 0, "", "", false
+ }
+ return id, filename, rel, true
+}
+
+func removeThumbnail(filename string) {
+ thumb := filepath.Join(Cfg.UploadDir, "thumbnails", filename+".jpg")
+ if err := os.Remove(thumb); err != nil && !errors.Is(err, os.ErrNotExist) {
+ log.Printf("Warning: removeThumbnail: %s: %v", thumb, err)
+ }
+}
+
+// remove DB records for removed files in upload dir
+func forgetUploadRecord(absPath string) {
+ id, filename, _, ok := uploadRecord(absPath)
+ if !ok {
+ return
+ }
+ tx, err := DB.Begin()
+ if err != nil {
+ log.Printf("Warning: forgetUploadRecord: failed to start transaction for %s: %v", absPath, err)
+ return
+ }
+ defer tx.Rollback()
+ for _, q := range []string{
+ "DELETE FROM file_tags WHERE file_id = ?",
+ "DELETE FROM file_properties WHERE file_id = ?",
+ "DELETE FROM files WHERE id = ?",
+ } {
+ if _, err := tx.Exec(q, id); err != nil {
+ log.Printf("Warning: forgetUploadRecord: %s (id=%d): %v", q, id, err)
+ return
+ }
+ }
+ if err := tx.Commit(); err != nil {
+ log.Printf("Warning: forgetUploadRecord: commit failed for id=%d: %v", id, err)
+ return
+ }
+ removeThumbnail(filename)
+}
+
+func syncUploadRecordsAfterReplace(src, dst, dest string) {
+ forgetUploadRecord(src)
+
+ id, filename, _, ok := uploadRecord(dst)
+ if !ok {
+ return
+ }
+
+ if dest != dst {
+ uploadAbs, err := filepath.Abs(Cfg.UploadDir)
+ if err == nil {
+ if newRel, err := filepath.Rel(uploadAbs, dest); err == nil {
+ newName := filepath.Base(dest)
+ removeThumbnail(filename)
+ if _, err := DB.Exec("UPDATE files SET filename = ?, path = ? WHERE id = ?", newName, newRel, id); err != nil {
+ log.Printf("Warning: syncUploadRecordsAfterReplace: failed to update record id=%d: %v", id, err)
+ }
+ }
+ }
+ } else {
+ removeThumbnail(filename) // its content changed, so the old thumbnail is stale
+ }
+
+ if _, err := DB.Exec("DELETE FROM file_properties WHERE file_id = ?", id); err != nil {
+ log.Printf("Warning: syncUploadRecordsAfterReplace: failed to clear properties for id=%d: %v", id, err)
+ }
+ computeProperties(id, dest)
+}
diff --git a/internal/app/routes.go b/internal/app/routes.go
@@ -12,6 +12,12 @@ func RegisterRoutes() {
http.HandleFunc("/add-yt", ytdlpHandler)
http.HandleFunc("/add-local", localFileHandler)
http.HandleFunc("/admin", adminHandler)
+ http.HandleFunc("/admin/duplicates/delete", duplicatesDeleteHandler)
+ http.HandleFunc("/admin/duplicates/delete-group", duplicatesDeleteGroupHandler)
+ http.HandleFunc("/admin/duplicates/groups", duplicatesGroupsHandler)
+ http.HandleFunc("/admin/duplicates/image", duplicatesImageHandler)
+ http.HandleFunc("/admin/duplicates/replace", duplicatesReplaceHandler)
+ http.HandleFunc("/admin/duplicates/scan", duplicatesScanHandler)
http.HandleFunc("/bulk-tag", bulkTagHandler)
http.HandleFunc("/cbz/", cbzViewerHandler)
http.HandleFunc("/file/", fileRouter)
diff --git a/main.go b/main.go
@@ -15,7 +15,9 @@ func main() {
// CLI flags
dataDir := flag.String("d", ".", "Data directory (stores tagliatelle.db and uploads/ subfolder)")
port := flag.String("p", "8080", "Port to listen on")
+ czkawka := flag.String("czkawka", "czkawka_cli", "Path to the czkawka_cli binary (used by the admin Duplicates tab)")
flag.Parse()
+ app.CzkawkaBin = *czkawka
// Derive paths from -d
dbPath := filepath.Join(*dataDir, "tagliatelle.db")
diff --git a/readme.md b/readme.md
@@ -17,6 +17,7 @@ go run . -d your_directory -p 8080
Then access the server via a web browser, the default port is 8080.
### Dependencies
+* `czkawka` for similar duplicate reporting
* `ffprobe` and `ffmpeg` for converting incompatible video uploads
* `sed` for custom bulk note manipulation
* `yt-dlp` for video downloader
@@ -38,6 +39,7 @@ Then access the server via a web browser, the default port is 8080.
* In browser file management (delete, rename)
* Self-organising, categorised notes, with optional `sed` operation rules
* Orphan and reverse orphan finding
+* Duplicate reporting and management via [czkawka](https://github.com/qarmin/czkawka).
* Database backup and vacuum support
* `tag=!`, `tag=!123` and `tag=x,value=!` for duplicating previously applied tags
* Chainable `/and/tag/tag2/value2` and `/and/tag/tag2/previews` filter matching
@@ -49,8 +51,7 @@ Then access the server via a web browser, the default port is 8080.
## Future plans
* Detect `mpegts` videos and repair them.
-* Implement [jdupes](https://codeberg.org/jbruchon/jdupes) and [czkawka](https://github.com/qarmin/czkawka) duplicate reporting.
## Credits
* [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) for the go SQLite3 library
-* [Fluent UI System Icons](https://icones.js.org/collection/fluent) for SVG icons used
-\ No newline at end of file
+* [Fluent UI System Icons](https://icones.js.org/collection/fluent) for SVG icons used
diff --git a/static/admin-duplicates.js b/static/admin-duplicates.js
@@ -0,0 +1,224 @@
+(function () {
+ const $ = (s) => document.querySelector(s);
+
+ function updateCount() {
+ const all = document.querySelectorAll('#dup-groups .dup-group');
+ const total = all.length;
+ const shown = [...all].filter((g) => !g.hidden).length;
+ const label = total === 1 ? ' group' : ' groups';
+ $('#dup-count').textContent = shown === total ? total + label : shown + ' of ' + total + label;
+ const empty = $('#dup-empty');
+ empty.textContent = total === 0 ? 'No similar images found.' : 'No groups match the filter.';
+ empty.hidden = shown !== 0;
+ }
+
+ // filter results: space separated, case-insensitively against paths with -negation support
+ function applyFilter() {
+ const terms = $('#dup-filter').value.toLowerCase().split(/\s+/).filter(Boolean);
+ const include = terms.filter((t) => t[0] !== '-').map((t) => t);
+ const exclude = terms.filter((t) => t[0] === '-' && t.length > 1).map((t) => t.slice(1));
+ document.querySelectorAll('#dup-groups .dup-group').forEach((g) => {
+ const paths = [...g.querySelectorAll('.dup-card')].map((c) => c._item.path.toLowerCase());
+ const ok = include.every((t) => paths.some((p) => p.includes(t))) &&
+ !exclude.some((t) => paths.some((p) => p.includes(t)));
+ g.hidden = !ok;
+ });
+ updateCount();
+ }
+
+ // highlight matching part of path in groups
+ function highlightGroup(groupEl) {
+ const cards = [...groupEl.querySelectorAll('.dup-card')];
+ const paths = cards.map((c) => c._item.path);
+ let pre = 0, suf = 0;
+ if (paths.length >= 2) {
+ const min = Math.min(...paths.map((p) => p.length));
+ while (pre < min && paths.every((p) => p[pre] === paths[0][pre])) pre++;
+ while (pre + suf < min && paths.every((p) => p[p.length - 1 - suf] === paths[0][paths[0].length - 1 - suf])) suf++;
+ }
+ cards.forEach((c) => {
+ const p = c._item.path;
+ const el = c.querySelector('.dup-path');
+ el.textContent = '';
+ const parts = [[p.slice(0, pre), true], [p.slice(pre, p.length - suf), false], [p.slice(p.length - suf), true]];
+ for (const [text, match] of parts) {
+ if (!text) continue;
+ const span = document.createElement('span');
+ if (match) span.className = 'dup-match';
+ span.textContent = text;
+ el.append(span);
+ }
+ });
+ }
+
+ let msgTimer;
+ function showMsg(msg, isErr) {
+ const el = $('#dup-msg');
+ el.textContent = msg;
+ el.className = isErr ? 'dup-msg--err' : 'dup-msg--ok';
+ clearTimeout(msgTimer);
+ msgTimer = setTimeout(() => { el.textContent = ''; }, 8000);
+ }
+ const showErr = (msg) => showMsg(msg, true);
+
+ async function post(url, body) {
+ const res = await fetch(url, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ });
+ if (!res.ok) throw new Error((await res.text()).trim() || res.status);
+ return res;
+ }
+
+ // Tracks images in a group to only show replace for pairs
+ function syncGroup(groupEl) {
+ const n = groupEl.querySelectorAll('.dup-card').length;
+ if (n < 2) { groupEl.remove(); } else { groupEl.dataset.count = n; highlightGroup(groupEl); }
+ updateCount();
+ }
+
+ async function del(item, card, groupEl) {
+ if (!confirm('Permanently delete this file?\n\n' + item.path)) return;
+ try {
+ await post('/admin/duplicates/delete', { path: item.path });
+ card.remove();
+ syncGroup(groupEl);
+ } catch (e) {
+ showErr('Delete failed: ' + e.message);
+ }
+ }
+
+ async function delAll(groupEl) {
+ const items = [...groupEl.querySelectorAll('.dup-card')].map((c) => c._item);
+ if (!confirm('Permanently delete ALL ' + items.length + ' files in this group?\n\n' +
+ items.map((i) => i.path).join('\n') + '\n\nNo copy will be kept. This cannot be undone.')) return;
+ try {
+ await post('/admin/duplicates/delete-group', { paths: items.map((i) => i.path) });
+ showMsg('Deleted ' + items.length + ' files');
+ groupEl.remove();
+ updateCount();
+ } catch (e) {
+ showErr('Delete all failed: ' + e.message);
+ }
+ }
+
+ // replace files prompt
+ async function replaceOther(item, card, groupEl) {
+ const other = [...groupEl.querySelectorAll('.dup-card')].find((c) => c !== card);
+ if (!other) return;
+ const dst = other._item;
+ if (!confirm(
+ 'Replace this file:\n\n' + dst.path + ' (' + dst.dims + ', ' + dst.size + ')\n\n' +
+ 'with this one:\n\n' + item.path + ' (' + item.dims + ', ' + item.size + ')\n\n' +
+ 'The better file is moved to the first location and the original is overwritten. ' +
+ 'This cannot be undone.')) return;
+ try {
+ const res = await post('/admin/duplicates/replace', { src: item.path, dst: dst.path });
+ const { path } = await res.json();
+ showMsg('Replaced: ' + path);
+ groupEl.remove(); // only one file is left, so the group is resolved
+ updateCount();
+ } catch (e) {
+ showErr('Replace failed: ' + e.message);
+ }
+ }
+
+ function render(groups) {
+ const root = $('#dup-groups');
+ root.textContent = '';
+ groups.forEach((g, i) => {
+ const groupEl = document.createElement('section');
+ groupEl.className = 'dup-group';
+ groupEl.dataset.count = g.length;
+ const h = document.createElement('h3');
+ h.textContent = 'Group ' + (i + 1) + ' (' + g.length + ' images)';
+ const all = document.createElement('button');
+ all.type = 'button';
+ all.className = 'dup-btn dup-btn--delall'; all.textContent = 'Delete all';
+ all.title = 'Delete every file in this group';
+ all.onclick = () => delAll(groupEl);
+ h.append(all);
+ groupEl.append(h);
+
+ const row = document.createElement('div');
+ row.className = 'dup-row';
+ g.forEach((item) => {
+ const card = document.createElement('div');
+ card.className = 'dup-card' + (item.similarity === 'Original' ? ' dup-card--original' : '');
+ card._item = item;
+
+ const url = '/admin/duplicates/image?path=' + encodeURIComponent(item.path);
+ const a = document.createElement('a');
+ a.href = url; a.target = '_blank';
+ const img = document.createElement('img');
+ img.loading = 'lazy'; img.src = url; img.alt = '';
+ a.append(img);
+
+ const p = document.createElement('div');
+ p.className = 'dup-path'; p.textContent = item.path;
+
+ const meta = document.createElement('div');
+ meta.className = 'dup-meta';
+ for (const [cls, text] of [['dup-tag--sim', item.similarity], ['', item.dims], ['', item.size]]) {
+ const t = document.createElement('span');
+ t.className = 'dup-tag ' + cls; t.textContent = text;
+ meta.append(t);
+ }
+
+ const btn = document.createElement('button');
+ btn.type = 'button';
+ btn.className = 'dup-btn dup-btn--del'; btn.textContent = 'Delete';
+ btn.onclick = () => del(item, card, groupEl);
+
+ const rep = document.createElement('button');
+ rep.type = 'button';
+ rep.className = 'dup-btn dup-btn--replace'; rep.textContent = 'Replace other with this';
+ rep.title = 'Move this file over the other one in the pair';
+ rep.onclick = () => replaceOther(item, card, groupEl);
+
+ card.append(a, p, meta, btn, rep);
+ row.append(card);
+ });
+ groupEl.append(row);
+ root.append(groupEl);
+ highlightGroup(groupEl);
+ });
+ $('#dup-bar').hidden = false;
+ applyFilter();
+ }
+
+ async function scan(e) {
+ e.preventDefault();
+ const dirs = [$('#dup-dir1').value, $('#dup-dir2').value].map((d) => d.trim()).filter(Boolean);
+ const btn = $('#dup-scan');
+ const status = $('#dup-status');
+ btn.disabled = true;
+ status.textContent = 'Scanning… this can take a while for large directories.';
+ try {
+ const res = await post('/admin/duplicates/scan', { dirs });
+ const data = await res.json();
+ render(data.groups || []);
+ status.textContent = '';
+ } catch (err) {
+ status.textContent = '';
+ $('#dup-bar').hidden = false;
+ showErr('Scan failed: ' + err.message);
+ } finally {
+ btn.disabled = false;
+ }
+ }
+
+ document.addEventListener('DOMContentLoaded', function () {
+ $('#dup-form').addEventListener('submit', scan);
+ $('#dup-filter').addEventListener('input', applyFilter);
+ // Show the previous scan again after a reload (results are kept in memory only).
+ fetch('/admin/duplicates/groups').then((r) => r.json()).then((data) => {
+ if (data.dirs && data.dirs.length) {
+ $('#dup-dir1').value = data.dirs[0];
+ $('#dup-dir2').value = data.dirs[1] || '';
+ render(data.groups || []);
+ }
+ }).catch(() => {});
+ });
+})();
diff --git a/static/style.css b/static/style.css
@@ -265,3 +265,30 @@ div.config-split {flex: 1}
.admin-tip-box {margin-bottom:20px;padding:10px;background:#e7f3ff;border:1px solid #b3d9ff;border-radius:4px;max-width:500px}
.admin-regen-form {max-width:500px;padding:20px;background:#f8f9fa;border:1px solid #ddd;border-radius:5px}
.admin-mono-input {width:100%;padding:8px;font-size:14px;font-family:monospace}
+
+/* admin duplicates */
+.dup-form {max-width:600px}
+.dup-status {margin-left:12px;color:#666}
+.dup-bar {padding:10px 0;border-bottom:1px solid #ddd}
+.dup-filter {margin-left:12px;padding:6px 8px;width:380px;max-width:100%;font-size:14px}
+.dup-match {background:#fff3b0;color:#111;border-radius:2px}
+.dup-bar[hidden],.dup-empty[hidden] {display:none}
+#dup-msg {margin-left:12px}
+.dup-msg--ok {color:#2a9d4a}
+.dup-msg--err {color:#c0392b}
+.dup-empty {margin-top:30px;text-align:center;color:#666}
+.dup-group {border:1px solid #ccc;border-radius:6px;margin:16px 0;padding:10px}
+.dup-group > h3 {font-size:13px;margin:0 0 8px;color:#666}
+.dup-row {display:flex;flex-wrap:wrap;gap:12px;align-items:flex-start}
+.dup-card {width:340px;border:1px solid #ddd;border-radius:4px;padding:6px;box-sizing:border-box}
+.dup-card--original {border-color:#2a9d4a}
+.dup-card img {display:block;width:100%;height:300px;object-fit:contain;background:#eee;cursor:zoom-in}
+.dup-path {word-break:break-all;font-size:12px;margin:6px 0 2px;font-family:monospace}
+.dup-meta {font-size:12px;color:#555}
+.dup-tag {display:inline-block;padding:0 6px;border-radius:3px;background:#eee;margin-right:4px}
+.dup-card--original .dup-tag--sim {background:#2a9d4a;color:#fff}
+.dup-btn {margin-top:6px;padding:4px 10px;cursor:pointer}
+.dup-btn--del {color:#c0392b}
+.dup-btn--replace {margin-left:6px}
+.dup-btn--delall {float:right;margin:0 0 8px;color:#fff;background:#c0392b;border:1px solid #a93226;border-radius:3px}
+.dup-group:not([data-count="2"]) .dup-btn--replace {display:none}
diff --git a/templates/admin.html b/templates/admin.html
@@ -36,6 +36,9 @@
<button onclick="showAdminTab('thumbnails')" id="admin-tab-thumbnails" class="admin-tab-btn">
Thumbnails
</button>
+ <button onclick="showAdminTab('duplicates')" id="admin-tab-duplicates" class="admin-tab-btn">
+ Duplicates
+ </button>
</div>
<!-- Settings Tab -->
@@ -368,6 +371,40 @@
</div>
</div>
+<!-- Duplicates Tab -->
+<div id="admin-content-duplicates" class="admin-tab-content">
+ <h2>Duplicates</h2>
+ <p class="admin-hint">
+ Scan one or two directories with <code>czkawka_cli</code> for similar images, then compare, delete or replace them.
+ Paths are not saved. Files inside the uploads directory also have their database records updated.
+ </p>
+
+ <form id="dup-form" class="dup-form">
+ <div class="admin-form-group">
+ <label for="dup-dir1" class="admin-form-label">Directory 1:</label>
+ <input type="text" id="dup-dir1" class="admin-mono-input" value="{{.Data.Config.UploadDir}}" placeholder="/path/to/directory">
+ </div>
+ <div class="admin-form-group">
+ <label for="dup-dir2" class="admin-form-label">Directory 2 (optional):</label>
+ <input type="text" id="dup-dir2" class="admin-mono-input" value="" placeholder="/path/to/another/directory">
+ </div>
+ <button type="submit" id="dup-scan" class="admin-btn admin-btn--primary">Scan for duplicates</button>
+ <span id="dup-status" class="dup-status"></span>
+ </form>
+
+ <div id="dup-bar" class="dup-bar" hidden>
+ <strong id="dup-count"></strong>
+ <input type="search" id="dup-filter" class="dup-filter" placeholder="Filter paths… prefix a word with - to exclude (e.g. -results)" autocomplete="off">
+ <span id="dup-msg"></span>
+ </div>
+ <div id="dup-groups"></div>
+ <div id="dup-empty" class="dup-empty" hidden>No similar images found.</div>
+
+ <a href="#top" class="back-to-top" title="Return to top">
+ <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="m18 15-6-6-6 6"/></svg>
+ </a>
+</div>
+
<script>window.initialAliasGroups = {{.Data.Config.TagAliases}};</script>
<script>window.initialSedRules = {{.Data.Config.SedRules}};</script>
<script>window.initialMagicLinks = {{.Data.Config.MagicLinks}};</script>
@@ -375,6 +412,7 @@
<script src="/static/admin-tag-alias.js" defer></script>
<script src="/static/admin-sed-rules.js" defer></script>
<script src="/static/admin-magic-links.js" defer></script>
+<script src="/static/admin-duplicates.js" defer></script>
<script src="/static/admin-tabs.js"></script>
<script src="/static/common.js" defer></script>
<script src="/static/hover-preview.js" defer></script>
