Add export command and shared item tree

Introduces a pure, FUSE-independent item tree (buildItemTree) shared by the mount and export commands. The new export subcommand downloads Docspell search results into a local directory, mirroring the mount's friendly folder/year/month hierarchy without the by-id directory or metadata JSON file, for archival or migration.

- itemtree.go: pure item tree builder shared by mount and export
- mount.go: buildMountTree now converts the pure tree into FUSE nodes, with a shared fileNode per attachment across friendly and by-id paths
- export.go: export command with concurrent downloads, --dry-run, --skip-existing, --concurrency, --query, --limit, --auth-header
- main.go: wires up the export subcommand and its flags
- export_test.go: tests for plan collection, writes, skip-existing, failures, modtime
- README.md: documents the export command
This commit is contained in:
Jan Bader committed 2026-07-21 23:35:03 +02:00
1 parent c4a2e9d085
commit 5e88eaea97
6 files changed
+696 -98

No files matched your search

+20
View File
@@ -10,6 +10,7 @@ A small command-line helper for managing Docspell import folders, archiving file
- Preserve useful archive structure by Docspell folder and item date.
- Optionally upload files that do not yet exist in Docspell.
- Mount Docspell search results as a read-only filesystem.
- Export Docspell search results to a local directory for archival or migration.
- Resolve Docspell item URLs/IDs to local mounted paths, and local mounted paths back to Docspell item URLs.
## Requirements
@@ -86,6 +87,25 @@ DS_CC_UPLOAD_MISSING=true docspell-cli import
Uploaded files are not archived immediately. They are re-checked on the next run, after Docspell has processed and confirmed them.
## Export to a local directory
The `export` subcommand downloads Docspell search results into a local directory, reproducing the same folder/year/month hierarchy exposed by the `mount` command but without the `by-id/` directory or `.docspell-cli-metadata.json` file. This is useful for archival or migration to a plain on-disk layout.
The destination directory is a required positional argument; there is no config entry for it.
```bash
docspell-cli export ~/Documents/DocspellExport
```
Options:
- `--query`: Docspell item query to export. Defaults to `mountQuery` from the config.
- `--limit`: number of items fetched per API request. Defaults to `1000`.
- `--concurrency`: number of attachments to download in parallel. Defaults to `4`.
- `--dry-run`: print the files that would be written without downloading.
- `--skip-existing`: skip files that already exist in the destination.
- `--auth-header`: HTTP header used for the Docspell auth token. Defaults to `X-Docspell-Auth`.
## Read-only FUSE mount
The `mount` subcommand exposes Docspell search results as a read-only filesystem. It uses the same XDG config file and password command as the import/archive command.
+252
View File
@@ -0,0 +1,252 @@
package main
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"sync"
"sync/atomic"
"time"
"github.com/integrii/flaggy"
)
// byIDDirName is the directory name used by buildItemTree for the ID-based
// lookup tree. The export command mirrors only the friendly hierarchy, so it
// skips this directory.
const byIDDirName = "by-id"
// runExportCommand downloads Docspell search results into a local directory,
// reproducing the same folder/year/month/<name> hierarchy exposed by the mount
// command but without the by-id directory or the metadata JSON file. This is
// useful for archival or migration to a plain on-disk layout.
func runExportCommand(args []string, cfg config, password string) error {
options, err := parseExportOptions(args, cfg)
if err != nil {
return err
}
if options.destination == "" {
return fmt.Errorf("export destination missing: pass a destination directory as the first positional argument")
}
absDest, err := filepath.Abs(options.destination)
if err != nil {
return fmt.Errorf("resolve destination: %w", err)
}
if info, err := os.Stat(absDest); err == nil && !info.IsDir() {
return fmt.Errorf("destination %s exists and is not a directory", absDest)
}
ctx := context.Background()
client := NewDocspellClient(cfg.DocspellURL, options.authHeader)
if err := client.Login(ctx, cfg.User, password); err != nil {
return fmt.Errorf("login: %w", err)
}
fmt.Printf("Searching Docspell items (query=%q, limit=%d)...\n", options.query, options.limit)
items, err := client.SearchItems(ctx, options.query, options.limit)
if err != nil {
return fmt.Errorf("search items: %w", err)
}
root, _ := buildItemTree(items)
plan := collectExportPlan(root)
if options.dryRun {
fmt.Printf("Dry run: %d file(s) would be written under %s\n", len(plan), absDest)
for _, entry := range plan {
fmt.Println(" ", filepath.FromSlash(entry.relPath))
}
return nil
}
fmt.Printf("Exporting %d file(s) to %s (concurrency=%d)...\n", len(plan), absDest, options.concurrency)
if err := os.MkdirAll(absDest, 0755); err != nil {
return fmt.Errorf("create destination: %w", err)
}
download := func(ctx context.Context, attachmentID string) ([]byte, error) {
return client.DownloadAttachment(ctx, attachmentID)
}
stats, err := executeExportPlan(ctx, plan, absDest, download, options.concurrency, options.skipExisting, os.Stdout)
if err != nil {
return err
}
fmt.Printf("Export complete: %d written, %d skipped, %d failed out of %d total.\n",
stats.written, stats.skipped, stats.failed, len(plan))
if stats.failed > 0 {
return fmt.Errorf("%d file(s) failed to export", stats.failed)
}
return nil
}
type exportOptions struct {
query string
limit int
authHeader string
destination string
concurrency int
dryRun bool
skipExisting bool
}
func parseExportOptions(args []string, cfg config) (exportOptions, error) {
options := exportOptions{
query: cfg.MountQuery,
limit: 1000,
authHeader: defaultDocspellAuthHeader,
concurrency: 4,
}
parser := flaggy.NewParser("export")
parser.Description = "Download Docspell search results into a local directory, mirroring the mount hierarchy"
parser.DisableShowVersionWithVersion()
parser.ShowHelpOnUnexpected = true
parser.String(&options.query, "", "query", "Docspell item query to expose")
parser.Int(&options.limit, "", "limit", "number of items to fetch per API request")
parser.String(&options.authHeader, "", "auth-header", "HTTP header used for the Docspell auth token")
parser.Int(&options.concurrency, "", "concurrency", "number of attachments to download in parallel")
parser.Bool(&options.dryRun, "", "dry-run", "print the files that would be written without downloading")
parser.Bool(&options.skipExisting, "", "skip-existing", "skip files that already exist in the destination")
parser.AddPositionalValue(&options.destination, "destination", 1, false, "destination directory for the export")
return options, parser.ParseArgs(args)
}
// exportPlanEntry describes a single file to write into the export destination.
// relPath is slash-separated and relative to the destination root.
type exportPlanEntry struct {
relPath string
attachmentID string
modTime int64 // unix seconds; 0 means preserve current time
}
// collectExportPlan walks the friendly hierarchy of the item tree (skipping the
// by-id directory and any metadata file) and returns the files to write in a
// deterministic, depth-first, name-sorted order.
func collectExportPlan(root *itemDir) []exportPlanEntry {
var plan []exportPlanEntry
for name, child := range root.children {
if name == byIDDirName || name == mountMetadataFileName {
continue
}
collectExportPlanNode(child, name, &plan)
}
return plan
}
func collectExportPlanNode(node itemNode, relPath string, plan *[]exportPlanEntry) {
switch n := node.(type) {
case *itemDir:
names := make([]string, 0, len(n.children))
for name := range n.children {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
collectExportPlanNode(n.children[name], relPath+"/"+name, plan)
}
case *itemFile:
modTime := int64(0)
if !n.modTime.IsZero() {
modTime = n.modTime.Unix()
}
*plan = append(*plan, exportPlanEntry{
relPath: relPath,
attachmentID: n.attachmentID,
modTime: modTime,
})
}
}
type exportStats struct {
written int64
skipped int64
failed int64
}
func (s *exportStats) addWritten() { atomic.AddInt64(&s.written, 1) }
func (s *exportStats) addSkipped() { atomic.AddInt64(&s.skipped, 1) }
func (s *exportStats) addFailed() { atomic.AddInt64(&s.failed, 1) }
// downloadFunc fetches an attachment by ID. It is a parameter so tests can
// substitute an in-memory implementation.
type downloadFunc func(ctx context.Context, attachmentID string) ([]byte, error)
// executeExportPlan downloads and writes every entry in plan under dest. Up to
// concurrency attachments are fetched in parallel. Errors from individual
// attachments are reported but do not abort the whole run.
func executeExportPlan(ctx context.Context, plan []exportPlanEntry, dest string, download downloadFunc, concurrency int, skipExisting bool, out io.Writer) (exportStats, error) {
if concurrency < 1 {
concurrency = 1
}
jobs := make(chan exportPlanEntry)
var stats exportStats
var reasonsMu sync.Mutex
var failedReasons []string
var wg sync.WaitGroup
for w := 0; w < concurrency; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for entry := range jobs {
switch err := exportOne(ctx, entry, dest, download, skipExisting); {
case errors.Is(err, errSkipExisting):
stats.addSkipped()
fmt.Fprintln(out, " skip", filepath.FromSlash(entry.relPath))
case err != nil:
stats.addFailed()
reasonsMu.Lock()
failedReasons = append(failedReasons, fmt.Sprintf("%s: %v", filepath.FromSlash(entry.relPath), err))
reasonsMu.Unlock()
fmt.Fprintln(out, " FAIL", filepath.FromSlash(entry.relPath), "-", err)
default:
stats.addWritten()
fmt.Fprintln(out, " wrote", filepath.FromSlash(entry.relPath))
}
}
}()
}
for _, entry := range plan {
jobs <- entry
}
close(jobs)
wg.Wait()
if len(failedReasons) > 0 {
fmt.Fprintln(out, "Failed files:")
for _, reason := range failedReasons {
fmt.Fprintln(out, " -", reason)
}
}
return stats, nil
}
var errSkipExisting = errors.New("skip existing file")
func exportOne(ctx context.Context, entry exportPlanEntry, dest string, download downloadFunc, skipExisting bool) error {
target := filepath.Join(dest, filepath.FromSlash(entry.relPath))
if skipExisting {
if _, err := os.Stat(target); err == nil {
return errSkipExisting
}
}
data, err := download(ctx, entry.attachmentID)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
return err
}
if err := os.WriteFile(target, data, 0644); err != nil {
return err
}
if entry.modTime != 0 {
t := time.Unix(entry.modTime, 0)
_ = os.Chtimes(target, t, t)
}
return nil
}
+214
View File
@@ -0,0 +1,214 @@
package main
import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestCollectExportPlanMirrorsFriendlyHierarchy(t *testing.T) {
items := []SearchItem{{
ID: "item-1",
Name: "Invoice/July",
Date: 1720396800000, // 2024-07-08 UTC
Folder: &DocspellEntity{
Name: "Bills/Private",
},
Attachments: []ItemAttachment{
{ID: "att-1", Name: "invoice.pdf", Size: 123},
{ID: "att-2", Name: "scan.pdf", Size: 456},
},
}}
root, _ := buildItemTree(items)
plan := collectExportPlan(root)
want := []string{
"Bills_Private/2024/07/Invoice_July - 01.pdf",
"Bills_Private/2024/07/Invoice_July - 02.pdf",
}
if len(plan) != len(want) {
t.Fatalf("expected %d entries, got %d: %#v", len(want), len(plan), plan)
}
for i, e := range plan {
if e.relPath != want[i] {
t.Errorf("entry %d: expected %q, got %q", i, want[i], e.relPath)
}
}
if plan[0].attachmentID != "att-1" || plan[1].attachmentID != "att-2" {
t.Errorf("attachment ids not preserved: %+v", plan)
}
}
func TestCollectExportPlanSkipsByIDAndMetadata(t *testing.T) {
items := []SearchItem{{
ID: "item-1",
Name: "Invoice",
Attachments: []ItemAttachment{{ID: "att-1", Name: "invoice.pdf", Size: 1}},
}}
root, _ := buildItemTree(items)
plan := collectExportPlan(root)
if len(plan) != 1 {
t.Fatalf("expected a single friendly entry, got %d: %#v", len(plan), plan)
}
for _, e := range plan {
if strings.HasPrefix(e.relPath, "by-id/") {
t.Errorf("plan must not include by-id entries, got %q", e.relPath)
}
}
}
func TestCollectExportPlanItemsWithoutFolderOrDate(t *testing.T) {
items := []SearchItem{{
ID: "item-2",
Name: "Undated Note",
Attachments: []ItemAttachment{{ID: "att-3", Name: "note", Size: 1}},
}}
root, _ := buildItemTree(items)
plan := collectExportPlan(root)
if len(plan) != 1 || plan[0].relPath != "No Folder/No Date/Undated Note.pdf" {
t.Fatalf("unexpected plan: %#v", plan)
}
}
func TestExecuteExportPlanWritesFilesAndContent(t *testing.T) {
items := []SearchItem{{
ID: "item-1",
Name: "Invoice",
Date: 1720396800000,
Folder: &DocspellEntity{Name: "Bills"},
Attachments: []ItemAttachment{
{ID: "att-1", Name: "invoice.pdf", Size: 4},
{ID: "att-2", Name: "scan.pdf", Size: 4},
},
}}
root, _ := buildItemTree(items)
plan := collectExportPlan(root)
dest := t.TempDir()
contents := map[string]string{"att-1": "AAAA", "att-2": "BBBB"}
download := func(ctx context.Context, id string) ([]byte, error) {
return []byte(contents[id]), nil
}
var out bytes.Buffer
stats, err := executeExportPlan(context.Background(), plan, dest, download, 2, false, &out)
if err != nil {
t.Fatal(err)
}
if stats.written != 2 || stats.skipped != 0 || stats.failed != 0 {
t.Fatalf("unexpected stats: %+v", stats)
}
for rel, content := range map[string]string{
"Bills/2024/07/Invoice - 01.pdf": "AAAA",
"Bills/2024/07/Invoice - 02.pdf": "BBBB",
} {
got, err := os.ReadFile(filepath.Join(dest, filepath.FromSlash(rel)))
if err != nil {
t.Fatalf("read %s: %v", rel, err)
}
if string(got) != content {
t.Errorf("content for %s: expected %q, got %q", rel, content, got)
}
}
}
func TestExecuteExportPlanSkipExisting(t *testing.T) {
items := []SearchItem{{
ID: "item-1",
Name: "Invoice",
Attachments: []ItemAttachment{{ID: "att-1", Name: "invoice.pdf", Size: 4}},
}}
root, _ := buildItemTree(items)
plan := collectExportPlan(root)
dest := t.TempDir()
target := filepath.Join(dest, "No Folder", "No Date", "Invoice.pdf")
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(target, []byte("EXISTING"), 0644); err != nil {
t.Fatal(err)
}
download := func(ctx context.Context, id string) ([]byte, error) {
t.Fatal("download should not be called for skipped existing file")
return nil, nil
}
var out bytes.Buffer
stats, err := executeExportPlan(context.Background(), plan, dest, download, 1, true, &out)
if err != nil {
t.Fatal(err)
}
if stats.skipped != 1 || stats.written != 0 {
t.Fatalf("expected 1 skipped, 0 written, got %+v", stats)
}
got, _ := os.ReadFile(target)
if string(got) != "EXISTING" {
t.Fatalf("existing file should be untouched, got %q", got)
}
}
func TestExecuteExportPlanReportsFailures(t *testing.T) {
items := []SearchItem{{
ID: "item-1",
Name: "Invoice",
Attachments: []ItemAttachment{
{ID: "att-1", Name: "ok.pdf", Size: 1},
{ID: "att-2", Name: "bad.pdf", Size: 1},
},
}}
root, _ := buildItemTree(items)
plan := collectExportPlan(root)
dest := t.TempDir()
badErr := errors.New("boom")
download := func(ctx context.Context, id string) ([]byte, error) {
if id == "att-2" {
return nil, badErr
}
return []byte("X"), nil
}
var out bytes.Buffer
stats, err := executeExportPlan(context.Background(), plan, dest, download, 1, false, &out)
if err != nil {
t.Fatalf("expected no hard error from executeExportPlan, got %v", err)
}
if stats.written != 1 || stats.failed != 1 {
t.Fatalf("expected 1 written, 1 failed, got %+v", stats)
}
if !bytes.Contains(out.Bytes(), []byte("Failed files:")) {
t.Fatalf("expected failure summary in output, got:\n%s", out.String())
}
}
func TestExportOneSetsModTime(t *testing.T) {
mt := time.Unix(1700000000, 0)
plan := []exportPlanEntry{{
relPath: "Folder/2023/01/Note.pdf",
attachmentID: "att-1",
modTime: mt.Unix(),
}}
dest := t.TempDir()
download := func(ctx context.Context, id string) ([]byte, error) { return []byte("X"), nil }
if err := exportOne(context.Background(), plan[0], dest, download, false); err != nil {
t.Fatal(err)
}
info, err := os.Stat(filepath.Join(dest, "Folder", "2023", "01", "Note.pdf"))
if err != nil {
t.Fatal(err)
}
if !info.ModTime().Equal(mt) {
t.Fatalf("expected modtime %v, got %v", mt, info.ModTime())
}
}
+138
View File
@@ -0,0 +1,138 @@
package main
import (
"fmt"
"path/filepath"
"strings"
"time"
)
// itemNode is either an itemDir or an itemFile in the pure, FUSE-independent
// representation of the Docspell item hierarchy built by buildItemTree.
type itemNode interface{ isItemNode() }
// itemDir is a directory in the pure item tree. It mirrors the FUSE dirNode but
// carries no FUSE-specific state, so it can be reused by the mount and export
// commands without importing bazil.org/fuse.
type itemDir struct {
name string
children map[string]itemNode
}
func (*itemDir) isItemNode() {}
// itemFile is a file entry backed by a Docspell attachment. The same *itemFile
// pointer is referenced from both the friendly hierarchy and the by-id tree, so
// consumers (mount) can share a single backing node between both locations.
type itemFile struct {
name string
attachmentID string
size int64
modTime time.Time
}
func (*itemFile) isItemNode() {}
// buildItemTree builds the pure item tree for the given Docspell search items.
//
// The returned root contains the friendly hierarchy (folder/year/month/<name>)
// plus a "by-id" directory with stable ID-based lookups. Files in the friendly
// hierarchy and the by-id tree share the same *itemFile pointers. The returned
// metadata maps item IDs to their friendly and by-id relative paths.
//
// The metadata JSON file itself is intentionally not part of the tree; callers
// that need it (the mount command) add it from the returned metadata.
func buildItemTree(items []SearchItem) (root *itemDir, metadata mountMetadata) {
root = &itemDir{name: "", children: map[string]itemNode{}}
byID := &itemDir{name: "by-id", children: map[string]itemNode{}}
root.children["by-id"] = byID
metadata = mountMetadata{Items: map[string]mountedItemMetadata{}}
for _, item := range items {
folderName := "No Folder"
if item.Folder != nil && item.Folder.Name != "" {
folderName = item.Folder.Name
}
date := item.Date
if date == 0 {
date = item.ItemDate
}
yearMonth := "No Date"
modTime := time.Now()
if date > 0 {
modTime = time.UnixMilli(date)
yearMonth = modTime.Format("2006/01")
}
dirParts := append([]string{folderName}, strings.Split(yearMonth, "/")...)
dir := ensureItemDir(root, dirParts[0], dirParts[1:]...)
base := sanitizeName(item.Name)
if base == "" {
base = item.ID
}
for i, att := range item.Attachments {
ext := filepath.Ext(att.Name)
if ext == "" {
ext = ".pdf"
}
name := base + ext
if len(item.Attachments) > 1 {
name = fmt.Sprintf("%s - %02d%s", base, i+1, ext)
}
name = uniqueItemName(dir.children, sanitizeName(name))
file := &itemFile{name: name, attachmentID: att.ID, size: att.Size, modTime: modTime}
dir.children[name] = file
friendlyRelPath := filepath.ToSlash(filepath.Join(append(sanitizePathParts(dirParts), name)...))
byIDName := item.ID + ext
if len(item.Attachments) > 1 && i > 0 {
byIDName = fmt.Sprintf("%s-%d%s", item.ID, i+1, ext)
}
shard := idShard(item.ID)
shardDir := ensureItemDir(byID, shard)
byIDName = uniqueItemName(shardDir.children, sanitizeName(byIDName))
shardDir.children[byIDName] = file
if i == 0 {
metadata.Items[item.ID] = mountedItemMetadata{
FriendlyPath: friendlyRelPath,
ByIDPath: filepath.ToSlash(filepath.Join("by-id", shard, byIDName)),
}
}
}
}
return root, metadata
}
func ensureItemDir(root *itemDir, first string, rest ...string) *itemDir {
cur := root
parts := append([]string{first}, rest...)
for _, part := range parts {
part = sanitizeName(part)
if part == "" {
part = "_"
}
next, ok := cur.children[part].(*itemDir)
if !ok {
next = &itemDir{name: part, children: map[string]itemNode{}}
cur.children[part] = next
}
cur = next
}
return cur
}
func uniqueItemName(children map[string]itemNode, name string) string {
if _, ok := children[name]; !ok {
return name
}
ext := filepath.Ext(name)
base := strings.TrimSuffix(name, ext)
for i := 2; ; i++ {
candidate := fmt.Sprintf("%s (%d)%s", base, i, ext)
if _, ok := children[candidate]; !ok {
return candidate
}
}
}
+20 -1
View File
@@ -60,6 +60,12 @@ func main() {
fmt.Fprintln(os.Stderr, "mount failed:", err)
os.Exit(1)
}
case commands.exportCommand.Used:
cfg, password := loadConfigAndPassword()
if err := runExportCommand(os.Args[2:], cfg, password); err != nil {
fmt.Fprintln(os.Stderr, "export failed:", err)
os.Exit(1)
}
case commands.getPathCommand.Used:
cfg, password := loadConfigAndPassword()
if err := runGetPathCommand(os.Args[2:], cfg, password); err != nil {
@@ -80,6 +86,7 @@ func main() {
type cliCommands struct {
importCommand *flaggy.Subcommand
mountCommand *flaggy.Subcommand
exportCommand *flaggy.Subcommand
getPathCommand *flaggy.Subcommand
getURLCommand *flaggy.Subcommand
}
@@ -93,28 +100,40 @@ func newCLIParser() (*flaggy.Parser, cliCommands) {
commands := cliCommands{
importCommand: flaggy.NewSubcommand("import"),
mountCommand: flaggy.NewSubcommand("mount"),
exportCommand: flaggy.NewSubcommand("export"),
getPathCommand: flaggy.NewSubcommand("get-path"),
getURLCommand: flaggy.NewSubcommand("get-url"),
}
commands.importCommand.Description = "Scan configured import folders, archive confirmed files, and optionally upload missing files"
commands.mountCommand.Description = "Expose Docspell search results as a read-only FUSE filesystem"
commands.exportCommand.Description = "Download Docspell search results into a local directory, mirroring the mount hierarchy"
commands.getPathCommand.Description = "Resolve a Docspell item ID or URL to the local mounted path"
commands.getURLCommand.Description = "Resolve a local mounted path, item ID, or item URL to a Docspell web URL"
var uploadMissing bool
commands.importCommand.Bool(&uploadMissing, "", "upload-missing", "true, to upload files to docspell that do not yet exist there")
var query, mountPoint, itemIDOrURL, pathOrID string
var query, mountPoint, exportDest, itemIDOrURL, pathOrID string
var exportConcurrency int
var exportDryRun, exportSkipExisting bool
authHeader := defaultDocspellAuthHeader
limit := 1000
commands.mountCommand.String(&query, "", "query", "Docspell item query to expose")
commands.mountCommand.Int(&limit, "", "limit", "number of items to fetch per API request")
commands.mountCommand.String(&authHeader, "", "auth-header", "HTTP header used for the Docspell auth token")
commands.mountCommand.AddPositionalValue(&mountPoint, "mountpoint", 1, false, "mount point, overriding mountPath from config")
commands.exportCommand.String(&query, "", "query", "Docspell item query to expose")
commands.exportCommand.Int(&limit, "", "limit", "number of items to fetch per API request")
commands.exportCommand.String(&authHeader, "", "auth-header", "HTTP header used for the Docspell auth token")
commands.exportCommand.AddPositionalValue(&exportDest, "destination", 1, false, "destination directory for the export")
commands.exportCommand.Int(&exportConcurrency, "", "concurrency", "number of attachments to download in parallel")
commands.exportCommand.Bool(&exportDryRun, "", "dry-run", "print the files that would be written without downloading")
commands.exportCommand.Bool(&exportSkipExisting, "", "skip-existing", "skip files that already exist in the destination")
commands.getPathCommand.AddPositionalValue(&itemIDOrURL, "item-id-or-url", 1, true, "Docspell item ID or web URL")
commands.getURLCommand.AddPositionalValue(&pathOrID, "local-path-or-item-id", 1, true, "local mounted path, item ID, or item URL")
parser.AttachSubcommand(commands.importCommand, 1)
parser.AttachSubcommand(commands.mountCommand, 1)
parser.AttachSubcommand(commands.exportCommand, 1)
parser.AttachSubcommand(commands.getPathCommand, 1)
parser.AttachSubcommand(commands.getURLCommand, 1)
return parser, commands
+52 -97
View File
@@ -337,76 +337,63 @@ func itemIDFromLocalPath(path string, cfg config) (string, error) {
return "", fmt.Errorf("could not map mounted path to item id")
}
// buildMountTree converts the pure item tree produced by buildItemTree into the
// FUSE node tree served by the mount command. A single *fileNode is created per
// *itemFile (shared between the friendly hierarchy and by-id lookups) so that a
// downloaded attachment is cached once regardless of which path reads it.
func buildMountTree(ctx context.Context, client *DocspellClient, items []SearchItem) *dirNode {
root := &dirNode{name: "", children: map[string]fs.Node{}}
byID := &dirNode{name: "by-id", children: map[string]fs.Node{}}
root.children["by-id"] = byID
metadata := mountMetadata{Items: map[string]mountedItemMetadata{}}
metadataModTime := time.Now()
for _, item := range items {
folderName := "No Folder"
if item.Folder != nil && item.Folder.Name != "" {
folderName = item.Folder.Name
}
date := item.Date
if date == 0 {
date = item.ItemDate
}
yearMonth := "No Date"
modTime := time.Now()
if date > 0 {
modTime = time.UnixMilli(date)
yearMonth = modTime.Format("2006/01")
}
dirParts := append([]string{folderName}, strings.Split(yearMonth, "/")...)
dir := ensureDir(root, dirParts[0], dirParts[1:]...)
base := sanitizeName(item.Name)
if base == "" {
base = item.ID
}
for i, att := range item.Attachments {
ext := filepath.Ext(att.Name)
if ext == "" {
ext = ".pdf"
}
name := base + ext
if len(item.Attachments) > 1 {
name = fmt.Sprintf("%s - %02d%s", base, i+1, ext)
}
size := att.Size
if size <= 0 {
if headSize, err := client.AttachmentSize(ctx, att.ID); err == nil && headSize > 0 {
size = headSize
}
}
name = uniqueName(dir.children, sanitizeName(name))
node := &fileNode{name: name, attachmentID: att.ID, size: uint64(size), knownSize: size > 0, modTime: modTime, client: client}
dir.children[name] = node
friendlyRelPath := filepath.ToSlash(filepath.Join(append(sanitizePathParts(dirParts), name)...))
root, metadata := buildItemTree(items)
fuseRoot := convertItemDir(root, ctx, client, map[*itemFile]*fileNode{})
byIDName := item.ID + ext
if len(item.Attachments) > 1 {
if i > 0 {
byIDName = fmt.Sprintf("%s-%d%s", item.ID, i+1, ext)
}
}
shard := idShard(item.ID)
shardDir := ensureDir(byID, shard)
byIDName = uniqueName(shardDir.children, sanitizeName(byIDName))
shardDir.children[byIDName] = node
if i == 0 {
metadata.Items[item.ID] = mountedItemMetadata{
FriendlyPath: friendlyRelPath,
ByIDPath: filepath.ToSlash(filepath.Join("by-id", shard, byIDName)),
}
}
}
}
metadataData, err := json.MarshalIndent(metadata, "", " ")
if err == nil {
root.children[mountMetadataFileName] = &staticFileNode{name: mountMetadataFileName, data: append(metadataData, '\n'), modTime: metadataModTime}
fuseRoot.children[mountMetadataFileName] = &staticFileNode{
name: mountMetadataFileName,
data: append(metadataData, '\n'),
modTime: time.Now(),
}
return root
}
return fuseRoot
}
// convertItemDir recursively converts a pure itemDir into a FUSE dirNode. The
// fileCache ensures that the same *itemFile (referenced from both the friendly
// hierarchy and the by-id tree) maps to a single shared *fileNode.
func convertItemDir(dir *itemDir, ctx context.Context, client *DocspellClient, fileCache map[*itemFile]*fileNode) *dirNode {
node := &dirNode{name: dir.name, children: map[string]fs.Node{}}
for name, child := range dir.children {
switch c := child.(type) {
case *itemDir:
node.children[name] = convertItemDir(c, ctx, client, fileCache)
case *itemFile:
node.children[name] = convertItemFile(c, ctx, client, fileCache)
}
}
return node
}
func convertItemFile(file *itemFile, ctx context.Context, client *DocspellClient, fileCache map[*itemFile]*fileNode) *fileNode {
if cached, ok := fileCache[file]; ok {
return cached
}
size := file.size
knownSize := size > 0
if size <= 0 {
if headSize, err := client.AttachmentSize(ctx, file.attachmentID); err == nil && headSize > 0 {
size = headSize
knownSize = true
}
}
node := &fileNode{
name: file.name,
attachmentID: file.attachmentID,
size: uint64(size),
knownSize: knownSize,
modTime: file.modTime,
client: client,
}
fileCache[file] = node
return node
}
func sanitizePathParts(parts []string) []string {
@@ -432,24 +419,6 @@ func idShard(id string) string {
return "__"
}
func ensureDir(root *dirNode, first string, rest ...string) *dirNode {
cur := root
parts := append([]string{first}, rest...)
for _, part := range parts {
part = sanitizeName(part)
if part == "" {
part = "_"
}
next, ok := cur.children[part].(*dirNode)
if !ok {
next = &dirNode{name: part, children: map[string]fs.Node{}}
cur.children[part] = next
}
cur = next
}
return cur
}
func sanitizeName(s string) string {
s = strings.TrimSpace(s)
s = strings.ReplaceAll(s, "/", "_")
@@ -457,20 +426,6 @@ func sanitizeName(s string) string {
return s
}
func uniqueName(children map[string]fs.Node, name string) string {
if _, ok := children[name]; !ok {
return name
}
ext := filepath.Ext(name)
base := strings.TrimSuffix(name, ext)
for i := 2; ; i++ {
candidate := fmt.Sprintf("%s (%d)%s", base, i, ext)
if _, ok := children[candidate]; !ok {
return candidate
}
}
}
func (f *docspellFS) Root() (fs.Node, error) { return f.root, nil }
func (d *dirNode) Attr(ctx context.Context, a *fuse.Attr) error {