From 5e88eaea97116aabcdb0a5ec617dab9ada58d16c Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Tue, 21 Jul 2026 23:35:03 +0200 Subject: [PATCH] Add export command and shared item tree Introduces a pure, FUSE-independent item tree (buildItemTree) shared by the mount and export commands. The new export subcommand downloads Docspell search results into a local directory, mirroring the mount's friendly folder/year/month hierarchy without the by-id directory or metadata JSON file, for archival or migration. - itemtree.go: pure item tree builder shared by mount and export - mount.go: buildMountTree now converts the pure tree into FUSE nodes, with a shared fileNode per attachment across friendly and by-id paths - export.go: export command with concurrent downloads, --dry-run, --skip-existing, --concurrency, --query, --limit, --auth-header - main.go: wires up the export subcommand and its flags - export_test.go: tests for plan collection, writes, skip-existing, failures, modtime - README.md: documents the export command --- README.md | 20 ++++ export.go | 252 +++++++++++++++++++++++++++++++++++++++++++++++++ export_test.go | 214 +++++++++++++++++++++++++++++++++++++++++ itemtree.go | 138 +++++++++++++++++++++++++++ main.go | 21 ++++- mount.go | 149 ++++++++++------------------- 6 files changed, 696 insertions(+), 98 deletions(-) create mode 100644 export.go create mode 100644 export_test.go create mode 100644 itemtree.go diff --git a/README.md b/README.md index b214008..5e43684 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ A small command-line helper for managing Docspell import folders, archiving file - Preserve useful archive structure by Docspell folder and item date. - Optionally upload files that do not yet exist in Docspell. - Mount Docspell search results as a read-only filesystem. +- Export Docspell search results to a local directory for archival or migration. - Resolve Docspell item URLs/IDs to local mounted paths, and local mounted paths back to Docspell item URLs. ## Requirements @@ -86,6 +87,25 @@ DS_CC_UPLOAD_MISSING=true docspell-cli import Uploaded files are not archived immediately. They are re-checked on the next run, after Docspell has processed and confirmed them. +## Export to a local directory + +The `export` subcommand downloads Docspell search results into a local directory, reproducing the same folder/year/month hierarchy exposed by the `mount` command but without the `by-id/` directory or `.docspell-cli-metadata.json` file. This is useful for archival or migration to a plain on-disk layout. + +The destination directory is a required positional argument; there is no config entry for it. + +```bash +docspell-cli export ~/Documents/DocspellExport +``` + +Options: + +- `--query`: Docspell item query to export. Defaults to `mountQuery` from the config. +- `--limit`: number of items fetched per API request. Defaults to `1000`. +- `--concurrency`: number of attachments to download in parallel. Defaults to `4`. +- `--dry-run`: print the files that would be written without downloading. +- `--skip-existing`: skip files that already exist in the destination. +- `--auth-header`: HTTP header used for the Docspell auth token. Defaults to `X-Docspell-Auth`. + ## Read-only FUSE mount The `mount` subcommand exposes Docspell search results as a read-only filesystem. It uses the same XDG config file and password command as the import/archive command. diff --git a/export.go b/export.go new file mode 100644 index 0000000..9dd4125 --- /dev/null +++ b/export.go @@ -0,0 +1,252 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/integrii/flaggy" +) + +// byIDDirName is the directory name used by buildItemTree for the ID-based +// lookup tree. The export command mirrors only the friendly hierarchy, so it +// skips this directory. +const byIDDirName = "by-id" + +// runExportCommand downloads Docspell search results into a local directory, +// reproducing the same folder/year/month/ hierarchy exposed by the mount +// command but without the by-id directory or the metadata JSON file. This is +// useful for archival or migration to a plain on-disk layout. +func runExportCommand(args []string, cfg config, password string) error { + options, err := parseExportOptions(args, cfg) + if err != nil { + return err + } + if options.destination == "" { + return fmt.Errorf("export destination missing: pass a destination directory as the first positional argument") + } + + absDest, err := filepath.Abs(options.destination) + if err != nil { + return fmt.Errorf("resolve destination: %w", err) + } + if info, err := os.Stat(absDest); err == nil && !info.IsDir() { + return fmt.Errorf("destination %s exists and is not a directory", absDest) + } + + ctx := context.Background() + client := NewDocspellClient(cfg.DocspellURL, options.authHeader) + if err := client.Login(ctx, cfg.User, password); err != nil { + return fmt.Errorf("login: %w", err) + } + + fmt.Printf("Searching Docspell items (query=%q, limit=%d)...\n", options.query, options.limit) + items, err := client.SearchItems(ctx, options.query, options.limit) + if err != nil { + return fmt.Errorf("search items: %w", err) + } + root, _ := buildItemTree(items) + plan := collectExportPlan(root) + + if options.dryRun { + fmt.Printf("Dry run: %d file(s) would be written under %s\n", len(plan), absDest) + for _, entry := range plan { + fmt.Println(" ", filepath.FromSlash(entry.relPath)) + } + return nil + } + + fmt.Printf("Exporting %d file(s) to %s (concurrency=%d)...\n", len(plan), absDest, options.concurrency) + if err := os.MkdirAll(absDest, 0755); err != nil { + return fmt.Errorf("create destination: %w", err) + } + download := func(ctx context.Context, attachmentID string) ([]byte, error) { + return client.DownloadAttachment(ctx, attachmentID) + } + stats, err := executeExportPlan(ctx, plan, absDest, download, options.concurrency, options.skipExisting, os.Stdout) + if err != nil { + return err + } + fmt.Printf("Export complete: %d written, %d skipped, %d failed out of %d total.\n", + stats.written, stats.skipped, stats.failed, len(plan)) + if stats.failed > 0 { + return fmt.Errorf("%d file(s) failed to export", stats.failed) + } + return nil +} + +type exportOptions struct { + query string + limit int + authHeader string + destination string + concurrency int + dryRun bool + skipExisting bool +} + +func parseExportOptions(args []string, cfg config) (exportOptions, error) { + options := exportOptions{ + query: cfg.MountQuery, + limit: 1000, + authHeader: defaultDocspellAuthHeader, + concurrency: 4, + } + parser := flaggy.NewParser("export") + parser.Description = "Download Docspell search results into a local directory, mirroring the mount hierarchy" + parser.DisableShowVersionWithVersion() + parser.ShowHelpOnUnexpected = true + parser.String(&options.query, "", "query", "Docspell item query to expose") + parser.Int(&options.limit, "", "limit", "number of items to fetch per API request") + parser.String(&options.authHeader, "", "auth-header", "HTTP header used for the Docspell auth token") + parser.Int(&options.concurrency, "", "concurrency", "number of attachments to download in parallel") + parser.Bool(&options.dryRun, "", "dry-run", "print the files that would be written without downloading") + parser.Bool(&options.skipExisting, "", "skip-existing", "skip files that already exist in the destination") + parser.AddPositionalValue(&options.destination, "destination", 1, false, "destination directory for the export") + return options, parser.ParseArgs(args) +} + +// exportPlanEntry describes a single file to write into the export destination. +// relPath is slash-separated and relative to the destination root. +type exportPlanEntry struct { + relPath string + attachmentID string + modTime int64 // unix seconds; 0 means preserve current time +} + +// collectExportPlan walks the friendly hierarchy of the item tree (skipping the +// by-id directory and any metadata file) and returns the files to write in a +// deterministic, depth-first, name-sorted order. +func collectExportPlan(root *itemDir) []exportPlanEntry { + var plan []exportPlanEntry + for name, child := range root.children { + if name == byIDDirName || name == mountMetadataFileName { + continue + } + collectExportPlanNode(child, name, &plan) + } + return plan +} + +func collectExportPlanNode(node itemNode, relPath string, plan *[]exportPlanEntry) { + switch n := node.(type) { + case *itemDir: + names := make([]string, 0, len(n.children)) + for name := range n.children { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + collectExportPlanNode(n.children[name], relPath+"/"+name, plan) + } + case *itemFile: + modTime := int64(0) + if !n.modTime.IsZero() { + modTime = n.modTime.Unix() + } + *plan = append(*plan, exportPlanEntry{ + relPath: relPath, + attachmentID: n.attachmentID, + modTime: modTime, + }) + } +} + +type exportStats struct { + written int64 + skipped int64 + failed int64 +} + +func (s *exportStats) addWritten() { atomic.AddInt64(&s.written, 1) } +func (s *exportStats) addSkipped() { atomic.AddInt64(&s.skipped, 1) } +func (s *exportStats) addFailed() { atomic.AddInt64(&s.failed, 1) } + +// downloadFunc fetches an attachment by ID. It is a parameter so tests can +// substitute an in-memory implementation. +type downloadFunc func(ctx context.Context, attachmentID string) ([]byte, error) + +// executeExportPlan downloads and writes every entry in plan under dest. Up to +// concurrency attachments are fetched in parallel. Errors from individual +// attachments are reported but do not abort the whole run. +func executeExportPlan(ctx context.Context, plan []exportPlanEntry, dest string, download downloadFunc, concurrency int, skipExisting bool, out io.Writer) (exportStats, error) { + if concurrency < 1 { + concurrency = 1 + } + + jobs := make(chan exportPlanEntry) + var stats exportStats + var reasonsMu sync.Mutex + var failedReasons []string + + var wg sync.WaitGroup + for w := 0; w < concurrency; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for entry := range jobs { + switch err := exportOne(ctx, entry, dest, download, skipExisting); { + case errors.Is(err, errSkipExisting): + stats.addSkipped() + fmt.Fprintln(out, " skip", filepath.FromSlash(entry.relPath)) + case err != nil: + stats.addFailed() + reasonsMu.Lock() + failedReasons = append(failedReasons, fmt.Sprintf("%s: %v", filepath.FromSlash(entry.relPath), err)) + reasonsMu.Unlock() + fmt.Fprintln(out, " FAIL", filepath.FromSlash(entry.relPath), "-", err) + default: + stats.addWritten() + fmt.Fprintln(out, " wrote", filepath.FromSlash(entry.relPath)) + } + } + }() + } + + for _, entry := range plan { + jobs <- entry + } + close(jobs) + wg.Wait() + + if len(failedReasons) > 0 { + fmt.Fprintln(out, "Failed files:") + for _, reason := range failedReasons { + fmt.Fprintln(out, " -", reason) + } + } + return stats, nil +} + +var errSkipExisting = errors.New("skip existing file") + +func exportOne(ctx context.Context, entry exportPlanEntry, dest string, download downloadFunc, skipExisting bool) error { + target := filepath.Join(dest, filepath.FromSlash(entry.relPath)) + if skipExisting { + if _, err := os.Stat(target); err == nil { + return errSkipExisting + } + } + data, err := download(ctx, entry.attachmentID) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + if err := os.WriteFile(target, data, 0644); err != nil { + return err + } + if entry.modTime != 0 { + t := time.Unix(entry.modTime, 0) + _ = os.Chtimes(target, t, t) + } + return nil +} diff --git a/export_test.go b/export_test.go new file mode 100644 index 0000000..14ea923 --- /dev/null +++ b/export_test.go @@ -0,0 +1,214 @@ +package main + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestCollectExportPlanMirrorsFriendlyHierarchy(t *testing.T) { + items := []SearchItem{{ + ID: "item-1", + Name: "Invoice/July", + Date: 1720396800000, // 2024-07-08 UTC + Folder: &DocspellEntity{ + Name: "Bills/Private", + }, + Attachments: []ItemAttachment{ + {ID: "att-1", Name: "invoice.pdf", Size: 123}, + {ID: "att-2", Name: "scan.pdf", Size: 456}, + }, + }} + + root, _ := buildItemTree(items) + plan := collectExportPlan(root) + + want := []string{ + "Bills_Private/2024/07/Invoice_July - 01.pdf", + "Bills_Private/2024/07/Invoice_July - 02.pdf", + } + if len(plan) != len(want) { + t.Fatalf("expected %d entries, got %d: %#v", len(want), len(plan), plan) + } + for i, e := range plan { + if e.relPath != want[i] { + t.Errorf("entry %d: expected %q, got %q", i, want[i], e.relPath) + } + } + if plan[0].attachmentID != "att-1" || plan[1].attachmentID != "att-2" { + t.Errorf("attachment ids not preserved: %+v", plan) + } +} + +func TestCollectExportPlanSkipsByIDAndMetadata(t *testing.T) { + items := []SearchItem{{ + ID: "item-1", + Name: "Invoice", + Attachments: []ItemAttachment{{ID: "att-1", Name: "invoice.pdf", Size: 1}}, + }} + + root, _ := buildItemTree(items) + plan := collectExportPlan(root) + + if len(plan) != 1 { + t.Fatalf("expected a single friendly entry, got %d: %#v", len(plan), plan) + } + for _, e := range plan { + if strings.HasPrefix(e.relPath, "by-id/") { + t.Errorf("plan must not include by-id entries, got %q", e.relPath) + } + } +} + +func TestCollectExportPlanItemsWithoutFolderOrDate(t *testing.T) { + items := []SearchItem{{ + ID: "item-2", + Name: "Undated Note", + Attachments: []ItemAttachment{{ID: "att-3", Name: "note", Size: 1}}, + }} + + root, _ := buildItemTree(items) + plan := collectExportPlan(root) + + if len(plan) != 1 || plan[0].relPath != "No Folder/No Date/Undated Note.pdf" { + t.Fatalf("unexpected plan: %#v", plan) + } +} + +func TestExecuteExportPlanWritesFilesAndContent(t *testing.T) { + items := []SearchItem{{ + ID: "item-1", + Name: "Invoice", + Date: 1720396800000, + Folder: &DocspellEntity{Name: "Bills"}, + Attachments: []ItemAttachment{ + {ID: "att-1", Name: "invoice.pdf", Size: 4}, + {ID: "att-2", Name: "scan.pdf", Size: 4}, + }, + }} + root, _ := buildItemTree(items) + plan := collectExportPlan(root) + + dest := t.TempDir() + contents := map[string]string{"att-1": "AAAA", "att-2": "BBBB"} + download := func(ctx context.Context, id string) ([]byte, error) { + return []byte(contents[id]), nil + } + + var out bytes.Buffer + stats, err := executeExportPlan(context.Background(), plan, dest, download, 2, false, &out) + if err != nil { + t.Fatal(err) + } + if stats.written != 2 || stats.skipped != 0 || stats.failed != 0 { + t.Fatalf("unexpected stats: %+v", stats) + } + + for rel, content := range map[string]string{ + "Bills/2024/07/Invoice - 01.pdf": "AAAA", + "Bills/2024/07/Invoice - 02.pdf": "BBBB", + } { + got, err := os.ReadFile(filepath.Join(dest, filepath.FromSlash(rel))) + if err != nil { + t.Fatalf("read %s: %v", rel, err) + } + if string(got) != content { + t.Errorf("content for %s: expected %q, got %q", rel, content, got) + } + } +} + +func TestExecuteExportPlanSkipExisting(t *testing.T) { + items := []SearchItem{{ + ID: "item-1", + Name: "Invoice", + Attachments: []ItemAttachment{{ID: "att-1", Name: "invoice.pdf", Size: 4}}, + }} + root, _ := buildItemTree(items) + plan := collectExportPlan(root) + + dest := t.TempDir() + target := filepath.Join(dest, "No Folder", "No Date", "Invoice.pdf") + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, []byte("EXISTING"), 0644); err != nil { + t.Fatal(err) + } + + download := func(ctx context.Context, id string) ([]byte, error) { + t.Fatal("download should not be called for skipped existing file") + return nil, nil + } + var out bytes.Buffer + stats, err := executeExportPlan(context.Background(), plan, dest, download, 1, true, &out) + if err != nil { + t.Fatal(err) + } + if stats.skipped != 1 || stats.written != 0 { + t.Fatalf("expected 1 skipped, 0 written, got %+v", stats) + } + got, _ := os.ReadFile(target) + if string(got) != "EXISTING" { + t.Fatalf("existing file should be untouched, got %q", got) + } +} + +func TestExecuteExportPlanReportsFailures(t *testing.T) { + items := []SearchItem{{ + ID: "item-1", + Name: "Invoice", + Attachments: []ItemAttachment{ + {ID: "att-1", Name: "ok.pdf", Size: 1}, + {ID: "att-2", Name: "bad.pdf", Size: 1}, + }, + }} + root, _ := buildItemTree(items) + plan := collectExportPlan(root) + + dest := t.TempDir() + badErr := errors.New("boom") + download := func(ctx context.Context, id string) ([]byte, error) { + if id == "att-2" { + return nil, badErr + } + return []byte("X"), nil + } + var out bytes.Buffer + stats, err := executeExportPlan(context.Background(), plan, dest, download, 1, false, &out) + if err != nil { + t.Fatalf("expected no hard error from executeExportPlan, got %v", err) + } + if stats.written != 1 || stats.failed != 1 { + t.Fatalf("expected 1 written, 1 failed, got %+v", stats) + } + if !bytes.Contains(out.Bytes(), []byte("Failed files:")) { + t.Fatalf("expected failure summary in output, got:\n%s", out.String()) + } +} + +func TestExportOneSetsModTime(t *testing.T) { + mt := time.Unix(1700000000, 0) + plan := []exportPlanEntry{{ + relPath: "Folder/2023/01/Note.pdf", + attachmentID: "att-1", + modTime: mt.Unix(), + }} + dest := t.TempDir() + download := func(ctx context.Context, id string) ([]byte, error) { return []byte("X"), nil } + if err := exportOne(context.Background(), plan[0], dest, download, false); err != nil { + t.Fatal(err) + } + info, err := os.Stat(filepath.Join(dest, "Folder", "2023", "01", "Note.pdf")) + if err != nil { + t.Fatal(err) + } + if !info.ModTime().Equal(mt) { + t.Fatalf("expected modtime %v, got %v", mt, info.ModTime()) + } +} diff --git a/itemtree.go b/itemtree.go new file mode 100644 index 0000000..6ea4f86 --- /dev/null +++ b/itemtree.go @@ -0,0 +1,138 @@ +package main + +import ( + "fmt" + "path/filepath" + "strings" + "time" +) + +// itemNode is either an itemDir or an itemFile in the pure, FUSE-independent +// representation of the Docspell item hierarchy built by buildItemTree. +type itemNode interface{ isItemNode() } + +// itemDir is a directory in the pure item tree. It mirrors the FUSE dirNode but +// carries no FUSE-specific state, so it can be reused by the mount and export +// commands without importing bazil.org/fuse. +type itemDir struct { + name string + children map[string]itemNode +} + +func (*itemDir) isItemNode() {} + +// itemFile is a file entry backed by a Docspell attachment. The same *itemFile +// pointer is referenced from both the friendly hierarchy and the by-id tree, so +// consumers (mount) can share a single backing node between both locations. +type itemFile struct { + name string + attachmentID string + size int64 + modTime time.Time +} + +func (*itemFile) isItemNode() {} + +// buildItemTree builds the pure item tree for the given Docspell search items. +// +// The returned root contains the friendly hierarchy (folder/year/month/) +// plus a "by-id" directory with stable ID-based lookups. Files in the friendly +// hierarchy and the by-id tree share the same *itemFile pointers. The returned +// metadata maps item IDs to their friendly and by-id relative paths. +// +// The metadata JSON file itself is intentionally not part of the tree; callers +// that need it (the mount command) add it from the returned metadata. +func buildItemTree(items []SearchItem) (root *itemDir, metadata mountMetadata) { + root = &itemDir{name: "", children: map[string]itemNode{}} + byID := &itemDir{name: "by-id", children: map[string]itemNode{}} + root.children["by-id"] = byID + metadata = mountMetadata{Items: map[string]mountedItemMetadata{}} + + for _, item := range items { + folderName := "No Folder" + if item.Folder != nil && item.Folder.Name != "" { + folderName = item.Folder.Name + } + date := item.Date + if date == 0 { + date = item.ItemDate + } + yearMonth := "No Date" + modTime := time.Now() + if date > 0 { + modTime = time.UnixMilli(date) + yearMonth = modTime.Format("2006/01") + } + dirParts := append([]string{folderName}, strings.Split(yearMonth, "/")...) + dir := ensureItemDir(root, dirParts[0], dirParts[1:]...) + + base := sanitizeName(item.Name) + if base == "" { + base = item.ID + } + + for i, att := range item.Attachments { + ext := filepath.Ext(att.Name) + if ext == "" { + ext = ".pdf" + } + name := base + ext + if len(item.Attachments) > 1 { + name = fmt.Sprintf("%s - %02d%s", base, i+1, ext) + } + name = uniqueItemName(dir.children, sanitizeName(name)) + file := &itemFile{name: name, attachmentID: att.ID, size: att.Size, modTime: modTime} + dir.children[name] = file + friendlyRelPath := filepath.ToSlash(filepath.Join(append(sanitizePathParts(dirParts), name)...)) + + byIDName := item.ID + ext + if len(item.Attachments) > 1 && i > 0 { + byIDName = fmt.Sprintf("%s-%d%s", item.ID, i+1, ext) + } + shard := idShard(item.ID) + shardDir := ensureItemDir(byID, shard) + byIDName = uniqueItemName(shardDir.children, sanitizeName(byIDName)) + shardDir.children[byIDName] = file + + if i == 0 { + metadata.Items[item.ID] = mountedItemMetadata{ + FriendlyPath: friendlyRelPath, + ByIDPath: filepath.ToSlash(filepath.Join("by-id", shard, byIDName)), + } + } + } + } + return root, metadata +} + +func ensureItemDir(root *itemDir, first string, rest ...string) *itemDir { + cur := root + parts := append([]string{first}, rest...) + for _, part := range parts { + part = sanitizeName(part) + if part == "" { + part = "_" + } + next, ok := cur.children[part].(*itemDir) + if !ok { + next = &itemDir{name: part, children: map[string]itemNode{}} + cur.children[part] = next + } + cur = next + } + return cur +} + +func uniqueItemName(children map[string]itemNode, name string) string { + if _, ok := children[name]; !ok { + return name + } + ext := filepath.Ext(name) + base := strings.TrimSuffix(name, ext) + for i := 2; ; i++ { + candidate := fmt.Sprintf("%s (%d)%s", base, i, ext) + if _, ok := children[candidate]; !ok { + return candidate + } + } +} diff --git a/main.go b/main.go index c87913f..4066fca 100644 --- a/main.go +++ b/main.go @@ -60,6 +60,12 @@ func main() { fmt.Fprintln(os.Stderr, "mount failed:", err) os.Exit(1) } + case commands.exportCommand.Used: + cfg, password := loadConfigAndPassword() + if err := runExportCommand(os.Args[2:], cfg, password); err != nil { + fmt.Fprintln(os.Stderr, "export failed:", err) + os.Exit(1) + } case commands.getPathCommand.Used: cfg, password := loadConfigAndPassword() if err := runGetPathCommand(os.Args[2:], cfg, password); err != nil { @@ -80,6 +86,7 @@ func main() { type cliCommands struct { importCommand *flaggy.Subcommand mountCommand *flaggy.Subcommand + exportCommand *flaggy.Subcommand getPathCommand *flaggy.Subcommand getURLCommand *flaggy.Subcommand } @@ -93,28 +100,40 @@ func newCLIParser() (*flaggy.Parser, cliCommands) { commands := cliCommands{ importCommand: flaggy.NewSubcommand("import"), mountCommand: flaggy.NewSubcommand("mount"), + exportCommand: flaggy.NewSubcommand("export"), getPathCommand: flaggy.NewSubcommand("get-path"), getURLCommand: flaggy.NewSubcommand("get-url"), } commands.importCommand.Description = "Scan configured import folders, archive confirmed files, and optionally upload missing files" commands.mountCommand.Description = "Expose Docspell search results as a read-only FUSE filesystem" + commands.exportCommand.Description = "Download Docspell search results into a local directory, mirroring the mount hierarchy" commands.getPathCommand.Description = "Resolve a Docspell item ID or URL to the local mounted path" commands.getURLCommand.Description = "Resolve a local mounted path, item ID, or item URL to a Docspell web URL" var uploadMissing bool commands.importCommand.Bool(&uploadMissing, "", "upload-missing", "true, to upload files to docspell that do not yet exist there") - var query, mountPoint, itemIDOrURL, pathOrID string + var query, mountPoint, exportDest, itemIDOrURL, pathOrID string + var exportConcurrency int + var exportDryRun, exportSkipExisting bool authHeader := defaultDocspellAuthHeader limit := 1000 commands.mountCommand.String(&query, "", "query", "Docspell item query to expose") commands.mountCommand.Int(&limit, "", "limit", "number of items to fetch per API request") commands.mountCommand.String(&authHeader, "", "auth-header", "HTTP header used for the Docspell auth token") commands.mountCommand.AddPositionalValue(&mountPoint, "mountpoint", 1, false, "mount point, overriding mountPath from config") + commands.exportCommand.String(&query, "", "query", "Docspell item query to expose") + commands.exportCommand.Int(&limit, "", "limit", "number of items to fetch per API request") + commands.exportCommand.String(&authHeader, "", "auth-header", "HTTP header used for the Docspell auth token") + commands.exportCommand.AddPositionalValue(&exportDest, "destination", 1, false, "destination directory for the export") + commands.exportCommand.Int(&exportConcurrency, "", "concurrency", "number of attachments to download in parallel") + commands.exportCommand.Bool(&exportDryRun, "", "dry-run", "print the files that would be written without downloading") + commands.exportCommand.Bool(&exportSkipExisting, "", "skip-existing", "skip files that already exist in the destination") commands.getPathCommand.AddPositionalValue(&itemIDOrURL, "item-id-or-url", 1, true, "Docspell item ID or web URL") commands.getURLCommand.AddPositionalValue(&pathOrID, "local-path-or-item-id", 1, true, "local mounted path, item ID, or item URL") parser.AttachSubcommand(commands.importCommand, 1) parser.AttachSubcommand(commands.mountCommand, 1) + parser.AttachSubcommand(commands.exportCommand, 1) parser.AttachSubcommand(commands.getPathCommand, 1) parser.AttachSubcommand(commands.getURLCommand, 1) return parser, commands diff --git a/mount.go b/mount.go index 3b4839f..52d5b37 100644 --- a/mount.go +++ b/mount.go @@ -337,76 +337,63 @@ func itemIDFromLocalPath(path string, cfg config) (string, error) { return "", fmt.Errorf("could not map mounted path to item id") } +// buildMountTree converts the pure item tree produced by buildItemTree into the +// FUSE node tree served by the mount command. A single *fileNode is created per +// *itemFile (shared between the friendly hierarchy and by-id lookups) so that a +// downloaded attachment is cached once regardless of which path reads it. func buildMountTree(ctx context.Context, client *DocspellClient, items []SearchItem) *dirNode { - root := &dirNode{name: "", children: map[string]fs.Node{}} - byID := &dirNode{name: "by-id", children: map[string]fs.Node{}} - root.children["by-id"] = byID - metadata := mountMetadata{Items: map[string]mountedItemMetadata{}} - metadataModTime := time.Now() - for _, item := range items { - folderName := "No Folder" - if item.Folder != nil && item.Folder.Name != "" { - folderName = item.Folder.Name - } - date := item.Date - if date == 0 { - date = item.ItemDate - } - yearMonth := "No Date" - modTime := time.Now() - if date > 0 { - modTime = time.UnixMilli(date) - yearMonth = modTime.Format("2006/01") - } - dirParts := append([]string{folderName}, strings.Split(yearMonth, "/")...) - dir := ensureDir(root, dirParts[0], dirParts[1:]...) - base := sanitizeName(item.Name) - if base == "" { - base = item.ID - } - for i, att := range item.Attachments { - ext := filepath.Ext(att.Name) - if ext == "" { - ext = ".pdf" - } - name := base + ext - if len(item.Attachments) > 1 { - name = fmt.Sprintf("%s - %02d%s", base, i+1, ext) - } - size := att.Size - if size <= 0 { - if headSize, err := client.AttachmentSize(ctx, att.ID); err == nil && headSize > 0 { - size = headSize - } - } - name = uniqueName(dir.children, sanitizeName(name)) - node := &fileNode{name: name, attachmentID: att.ID, size: uint64(size), knownSize: size > 0, modTime: modTime, client: client} - dir.children[name] = node - friendlyRelPath := filepath.ToSlash(filepath.Join(append(sanitizePathParts(dirParts), name)...)) + root, metadata := buildItemTree(items) + fuseRoot := convertItemDir(root, ctx, client, map[*itemFile]*fileNode{}) - byIDName := item.ID + ext - if len(item.Attachments) > 1 { - if i > 0 { - byIDName = fmt.Sprintf("%s-%d%s", item.ID, i+1, ext) - } - } - shard := idShard(item.ID) - shardDir := ensureDir(byID, shard) - byIDName = uniqueName(shardDir.children, sanitizeName(byIDName)) - shardDir.children[byIDName] = node - if i == 0 { - metadata.Items[item.ID] = mountedItemMetadata{ - FriendlyPath: friendlyRelPath, - ByIDPath: filepath.ToSlash(filepath.Join("by-id", shard, byIDName)), - } - } - } - } metadataData, err := json.MarshalIndent(metadata, "", " ") if err == nil { - root.children[mountMetadataFileName] = &staticFileNode{name: mountMetadataFileName, data: append(metadataData, '\n'), modTime: metadataModTime} + fuseRoot.children[mountMetadataFileName] = &staticFileNode{ + name: mountMetadataFileName, + data: append(metadataData, '\n'), + modTime: time.Now(), + } } - return root + return fuseRoot +} + +// convertItemDir recursively converts a pure itemDir into a FUSE dirNode. The +// fileCache ensures that the same *itemFile (referenced from both the friendly +// hierarchy and the by-id tree) maps to a single shared *fileNode. +func convertItemDir(dir *itemDir, ctx context.Context, client *DocspellClient, fileCache map[*itemFile]*fileNode) *dirNode { + node := &dirNode{name: dir.name, children: map[string]fs.Node{}} + for name, child := range dir.children { + switch c := child.(type) { + case *itemDir: + node.children[name] = convertItemDir(c, ctx, client, fileCache) + case *itemFile: + node.children[name] = convertItemFile(c, ctx, client, fileCache) + } + } + return node +} + +func convertItemFile(file *itemFile, ctx context.Context, client *DocspellClient, fileCache map[*itemFile]*fileNode) *fileNode { + if cached, ok := fileCache[file]; ok { + return cached + } + size := file.size + knownSize := size > 0 + if size <= 0 { + if headSize, err := client.AttachmentSize(ctx, file.attachmentID); err == nil && headSize > 0 { + size = headSize + knownSize = true + } + } + node := &fileNode{ + name: file.name, + attachmentID: file.attachmentID, + size: uint64(size), + knownSize: knownSize, + modTime: file.modTime, + client: client, + } + fileCache[file] = node + return node } func sanitizePathParts(parts []string) []string { @@ -432,24 +419,6 @@ func idShard(id string) string { return "__" } -func ensureDir(root *dirNode, first string, rest ...string) *dirNode { - cur := root - parts := append([]string{first}, rest...) - for _, part := range parts { - part = sanitizeName(part) - if part == "" { - part = "_" - } - next, ok := cur.children[part].(*dirNode) - if !ok { - next = &dirNode{name: part, children: map[string]fs.Node{}} - cur.children[part] = next - } - cur = next - } - return cur -} - func sanitizeName(s string) string { s = strings.TrimSpace(s) s = strings.ReplaceAll(s, "/", "_") @@ -457,20 +426,6 @@ func sanitizeName(s string) string { return s } -func uniqueName(children map[string]fs.Node, name string) string { - if _, ok := children[name]; !ok { - return name - } - ext := filepath.Ext(name) - base := strings.TrimSuffix(name, ext) - for i := 2; ; i++ { - candidate := fmt.Sprintf("%s (%d)%s", base, i, ext) - if _, ok := children[candidate]; !ok { - return candidate - } - } -} - func (f *docspellFS) Root() (fs.Node, error) { return f.root, nil } func (d *dirNode) Attr(ctx context.Context, a *fuse.Attr) error {