Add export command and shared item tree
Introduces a pure, FUSE-independent item tree (buildItemTree) shared by the mount and export commands. The new export subcommand downloads Docspell search results into a local directory, mirroring the mount's friendly folder/year/month hierarchy without the by-id directory or metadata JSON file, for archival or migration. - itemtree.go: pure item tree builder shared by mount and export - mount.go: buildMountTree now converts the pure tree into FUSE nodes, with a shared fileNode per attachment across friendly and by-id paths - export.go: export command with concurrent downloads, --dry-run, --skip-existing, --concurrency, --query, --limit, --auth-header - main.go: wires up the export subcommand and its flags - export_test.go: tests for plan collection, writes, skip-existing, failures, modtime - README.md: documents the export command
This commit is contained in:
1 parent
c4a2e9d085
commit
5e88eaea97
6 files changed
+696
-98
No files matched your search
@@ -10,6 +10,7 @@ A small command-line helper for managing Docspell import folders, archiving file
|
||||
- Preserve useful archive structure by Docspell folder and item date.
|
||||
- Optionally upload files that do not yet exist in Docspell.
|
||||
- Mount Docspell search results as a read-only filesystem.
|
||||
- Export Docspell search results to a local directory for archival or migration.
|
||||
- Resolve Docspell item URLs/IDs to local mounted paths, and local mounted paths back to Docspell item URLs.
|
||||
|
||||
## Requirements
|
||||
@@ -86,6 +87,25 @@ DS_CC_UPLOAD_MISSING=true docspell-cli import
|
||||
|
||||
Uploaded files are not archived immediately. They are re-checked on the next run, after Docspell has processed and confirmed them.
|
||||
|
||||
## Export to a local directory
|
||||
|
||||
The `export` subcommand downloads Docspell search results into a local directory, reproducing the same folder/year/month hierarchy exposed by the `mount` command but without the `by-id/` directory or `.docspell-cli-metadata.json` file. This is useful for archival or migration to a plain on-disk layout.
|
||||
|
||||
The destination directory is a required positional argument; there is no config entry for it.
|
||||
|
||||
```bash
|
||||
docspell-cli export ~/Documents/DocspellExport
|
||||
```
|
||||
|
||||
Options:
|
||||
|
||||
- `--query`: Docspell item query to export. Defaults to `mountQuery` from the config.
|
||||
- `--limit`: number of items fetched per API request. Defaults to `1000`.
|
||||
- `--concurrency`: number of attachments to download in parallel. Defaults to `4`.
|
||||
- `--dry-run`: print the files that would be written without downloading.
|
||||
- `--skip-existing`: skip files that already exist in the destination.
|
||||
- `--auth-header`: HTTP header used for the Docspell auth token. Defaults to `X-Docspell-Auth`.
|
||||
|
||||
## Read-only FUSE mount
|
||||
|
||||
The `mount` subcommand exposes Docspell search results as a read-only filesystem. It uses the same XDG config file and password command as the import/archive command.
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/integrii/flaggy"
|
||||
)
|
||||
|
||||
// byIDDirName is the directory name used by buildItemTree for the ID-based
|
||||
// lookup tree. The export command mirrors only the friendly hierarchy, so it
|
||||
// skips this directory.
|
||||
const byIDDirName = "by-id"
|
||||
|
||||
// runExportCommand downloads Docspell search results into a local directory,
|
||||
// reproducing the same folder/year/month/<name> hierarchy exposed by the mount
|
||||
// command but without the by-id directory or the metadata JSON file. This is
|
||||
// useful for archival or migration to a plain on-disk layout.
|
||||
func runExportCommand(args []string, cfg config, password string) error {
|
||||
options, err := parseExportOptions(args, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if options.destination == "" {
|
||||
return fmt.Errorf("export destination missing: pass a destination directory as the first positional argument")
|
||||
}
|
||||
|
||||
absDest, err := filepath.Abs(options.destination)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve destination: %w", err)
|
||||
}
|
||||
if info, err := os.Stat(absDest); err == nil && !info.IsDir() {
|
||||
return fmt.Errorf("destination %s exists and is not a directory", absDest)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
client := NewDocspellClient(cfg.DocspellURL, options.authHeader)
|
||||
if err := client.Login(ctx, cfg.User, password); err != nil {
|
||||
return fmt.Errorf("login: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Searching Docspell items (query=%q, limit=%d)...\n", options.query, options.limit)
|
||||
items, err := client.SearchItems(ctx, options.query, options.limit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("search items: %w", err)
|
||||
}
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
if options.dryRun {
|
||||
fmt.Printf("Dry run: %d file(s) would be written under %s\n", len(plan), absDest)
|
||||
for _, entry := range plan {
|
||||
fmt.Println(" ", filepath.FromSlash(entry.relPath))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("Exporting %d file(s) to %s (concurrency=%d)...\n", len(plan), absDest, options.concurrency)
|
||||
if err := os.MkdirAll(absDest, 0755); err != nil {
|
||||
return fmt.Errorf("create destination: %w", err)
|
||||
}
|
||||
download := func(ctx context.Context, attachmentID string) ([]byte, error) {
|
||||
return client.DownloadAttachment(ctx, attachmentID)
|
||||
}
|
||||
stats, err := executeExportPlan(ctx, plan, absDest, download, options.concurrency, options.skipExisting, os.Stdout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Export complete: %d written, %d skipped, %d failed out of %d total.\n",
|
||||
stats.written, stats.skipped, stats.failed, len(plan))
|
||||
if stats.failed > 0 {
|
||||
return fmt.Errorf("%d file(s) failed to export", stats.failed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type exportOptions struct {
|
||||
query string
|
||||
limit int
|
||||
authHeader string
|
||||
destination string
|
||||
concurrency int
|
||||
dryRun bool
|
||||
skipExisting bool
|
||||
}
|
||||
|
||||
func parseExportOptions(args []string, cfg config) (exportOptions, error) {
|
||||
options := exportOptions{
|
||||
query: cfg.MountQuery,
|
||||
limit: 1000,
|
||||
authHeader: defaultDocspellAuthHeader,
|
||||
concurrency: 4,
|
||||
}
|
||||
parser := flaggy.NewParser("export")
|
||||
parser.Description = "Download Docspell search results into a local directory, mirroring the mount hierarchy"
|
||||
parser.DisableShowVersionWithVersion()
|
||||
parser.ShowHelpOnUnexpected = true
|
||||
parser.String(&options.query, "", "query", "Docspell item query to expose")
|
||||
parser.Int(&options.limit, "", "limit", "number of items to fetch per API request")
|
||||
parser.String(&options.authHeader, "", "auth-header", "HTTP header used for the Docspell auth token")
|
||||
parser.Int(&options.concurrency, "", "concurrency", "number of attachments to download in parallel")
|
||||
parser.Bool(&options.dryRun, "", "dry-run", "print the files that would be written without downloading")
|
||||
parser.Bool(&options.skipExisting, "", "skip-existing", "skip files that already exist in the destination")
|
||||
parser.AddPositionalValue(&options.destination, "destination", 1, false, "destination directory for the export")
|
||||
return options, parser.ParseArgs(args)
|
||||
}
|
||||
|
||||
// exportPlanEntry describes a single file to write into the export destination.
|
||||
// relPath is slash-separated and relative to the destination root.
|
||||
type exportPlanEntry struct {
|
||||
relPath string
|
||||
attachmentID string
|
||||
modTime int64 // unix seconds; 0 means preserve current time
|
||||
}
|
||||
|
||||
// collectExportPlan walks the friendly hierarchy of the item tree (skipping the
|
||||
// by-id directory and any metadata file) and returns the files to write in a
|
||||
// deterministic, depth-first, name-sorted order.
|
||||
func collectExportPlan(root *itemDir) []exportPlanEntry {
|
||||
var plan []exportPlanEntry
|
||||
for name, child := range root.children {
|
||||
if name == byIDDirName || name == mountMetadataFileName {
|
||||
continue
|
||||
}
|
||||
collectExportPlanNode(child, name, &plan)
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
func collectExportPlanNode(node itemNode, relPath string, plan *[]exportPlanEntry) {
|
||||
switch n := node.(type) {
|
||||
case *itemDir:
|
||||
names := make([]string, 0, len(n.children))
|
||||
for name := range n.children {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
collectExportPlanNode(n.children[name], relPath+"/"+name, plan)
|
||||
}
|
||||
case *itemFile:
|
||||
modTime := int64(0)
|
||||
if !n.modTime.IsZero() {
|
||||
modTime = n.modTime.Unix()
|
||||
}
|
||||
*plan = append(*plan, exportPlanEntry{
|
||||
relPath: relPath,
|
||||
attachmentID: n.attachmentID,
|
||||
modTime: modTime,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type exportStats struct {
|
||||
written int64
|
||||
skipped int64
|
||||
failed int64
|
||||
}
|
||||
|
||||
func (s *exportStats) addWritten() { atomic.AddInt64(&s.written, 1) }
|
||||
func (s *exportStats) addSkipped() { atomic.AddInt64(&s.skipped, 1) }
|
||||
func (s *exportStats) addFailed() { atomic.AddInt64(&s.failed, 1) }
|
||||
|
||||
// downloadFunc fetches an attachment by ID. It is a parameter so tests can
|
||||
// substitute an in-memory implementation.
|
||||
type downloadFunc func(ctx context.Context, attachmentID string) ([]byte, error)
|
||||
|
||||
// executeExportPlan downloads and writes every entry in plan under dest. Up to
|
||||
// concurrency attachments are fetched in parallel. Errors from individual
|
||||
// attachments are reported but do not abort the whole run.
|
||||
func executeExportPlan(ctx context.Context, plan []exportPlanEntry, dest string, download downloadFunc, concurrency int, skipExisting bool, out io.Writer) (exportStats, error) {
|
||||
if concurrency < 1 {
|
||||
concurrency = 1
|
||||
}
|
||||
|
||||
jobs := make(chan exportPlanEntry)
|
||||
var stats exportStats
|
||||
var reasonsMu sync.Mutex
|
||||
var failedReasons []string
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < concurrency; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for entry := range jobs {
|
||||
switch err := exportOne(ctx, entry, dest, download, skipExisting); {
|
||||
case errors.Is(err, errSkipExisting):
|
||||
stats.addSkipped()
|
||||
fmt.Fprintln(out, " skip", filepath.FromSlash(entry.relPath))
|
||||
case err != nil:
|
||||
stats.addFailed()
|
||||
reasonsMu.Lock()
|
||||
failedReasons = append(failedReasons, fmt.Sprintf("%s: %v", filepath.FromSlash(entry.relPath), err))
|
||||
reasonsMu.Unlock()
|
||||
fmt.Fprintln(out, " FAIL", filepath.FromSlash(entry.relPath), "-", err)
|
||||
default:
|
||||
stats.addWritten()
|
||||
fmt.Fprintln(out, " wrote", filepath.FromSlash(entry.relPath))
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for _, entry := range plan {
|
||||
jobs <- entry
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
|
||||
if len(failedReasons) > 0 {
|
||||
fmt.Fprintln(out, "Failed files:")
|
||||
for _, reason := range failedReasons {
|
||||
fmt.Fprintln(out, " -", reason)
|
||||
}
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
var errSkipExisting = errors.New("skip existing file")
|
||||
|
||||
func exportOne(ctx context.Context, entry exportPlanEntry, dest string, download downloadFunc, skipExisting bool) error {
|
||||
target := filepath.Join(dest, filepath.FromSlash(entry.relPath))
|
||||
if skipExisting {
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
return errSkipExisting
|
||||
}
|
||||
}
|
||||
data, err := download(ctx, entry.attachmentID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(target, data, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.modTime != 0 {
|
||||
t := time.Unix(entry.modTime, 0)
|
||||
_ = os.Chtimes(target, t, t)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+214
@@ -0,0 +1,214 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCollectExportPlanMirrorsFriendlyHierarchy(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-1",
|
||||
Name: "Invoice/July",
|
||||
Date: 1720396800000, // 2024-07-08 UTC
|
||||
Folder: &DocspellEntity{
|
||||
Name: "Bills/Private",
|
||||
},
|
||||
Attachments: []ItemAttachment{
|
||||
{ID: "att-1", Name: "invoice.pdf", Size: 123},
|
||||
{ID: "att-2", Name: "scan.pdf", Size: 456},
|
||||
},
|
||||
}}
|
||||
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
want := []string{
|
||||
"Bills_Private/2024/07/Invoice_July - 01.pdf",
|
||||
"Bills_Private/2024/07/Invoice_July - 02.pdf",
|
||||
}
|
||||
if len(plan) != len(want) {
|
||||
t.Fatalf("expected %d entries, got %d: %#v", len(want), len(plan), plan)
|
||||
}
|
||||
for i, e := range plan {
|
||||
if e.relPath != want[i] {
|
||||
t.Errorf("entry %d: expected %q, got %q", i, want[i], e.relPath)
|
||||
}
|
||||
}
|
||||
if plan[0].attachmentID != "att-1" || plan[1].attachmentID != "att-2" {
|
||||
t.Errorf("attachment ids not preserved: %+v", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectExportPlanSkipsByIDAndMetadata(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-1",
|
||||
Name: "Invoice",
|
||||
Attachments: []ItemAttachment{{ID: "att-1", Name: "invoice.pdf", Size: 1}},
|
||||
}}
|
||||
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
if len(plan) != 1 {
|
||||
t.Fatalf("expected a single friendly entry, got %d: %#v", len(plan), plan)
|
||||
}
|
||||
for _, e := range plan {
|
||||
if strings.HasPrefix(e.relPath, "by-id/") {
|
||||
t.Errorf("plan must not include by-id entries, got %q", e.relPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectExportPlanItemsWithoutFolderOrDate(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-2",
|
||||
Name: "Undated Note",
|
||||
Attachments: []ItemAttachment{{ID: "att-3", Name: "note", Size: 1}},
|
||||
}}
|
||||
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
if len(plan) != 1 || plan[0].relPath != "No Folder/No Date/Undated Note.pdf" {
|
||||
t.Fatalf("unexpected plan: %#v", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteExportPlanWritesFilesAndContent(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-1",
|
||||
Name: "Invoice",
|
||||
Date: 1720396800000,
|
||||
Folder: &DocspellEntity{Name: "Bills"},
|
||||
Attachments: []ItemAttachment{
|
||||
{ID: "att-1", Name: "invoice.pdf", Size: 4},
|
||||
{ID: "att-2", Name: "scan.pdf", Size: 4},
|
||||
},
|
||||
}}
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
dest := t.TempDir()
|
||||
contents := map[string]string{"att-1": "AAAA", "att-2": "BBBB"}
|
||||
download := func(ctx context.Context, id string) ([]byte, error) {
|
||||
return []byte(contents[id]), nil
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
stats, err := executeExportPlan(context.Background(), plan, dest, download, 2, false, &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stats.written != 2 || stats.skipped != 0 || stats.failed != 0 {
|
||||
t.Fatalf("unexpected stats: %+v", stats)
|
||||
}
|
||||
|
||||
for rel, content := range map[string]string{
|
||||
"Bills/2024/07/Invoice - 01.pdf": "AAAA",
|
||||
"Bills/2024/07/Invoice - 02.pdf": "BBBB",
|
||||
} {
|
||||
got, err := os.ReadFile(filepath.Join(dest, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", rel, err)
|
||||
}
|
||||
if string(got) != content {
|
||||
t.Errorf("content for %s: expected %q, got %q", rel, content, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteExportPlanSkipExisting(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-1",
|
||||
Name: "Invoice",
|
||||
Attachments: []ItemAttachment{{ID: "att-1", Name: "invoice.pdf", Size: 4}},
|
||||
}}
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
dest := t.TempDir()
|
||||
target := filepath.Join(dest, "No Folder", "No Date", "Invoice.pdf")
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(target, []byte("EXISTING"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
download := func(ctx context.Context, id string) ([]byte, error) {
|
||||
t.Fatal("download should not be called for skipped existing file")
|
||||
return nil, nil
|
||||
}
|
||||
var out bytes.Buffer
|
||||
stats, err := executeExportPlan(context.Background(), plan, dest, download, 1, true, &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stats.skipped != 1 || stats.written != 0 {
|
||||
t.Fatalf("expected 1 skipped, 0 written, got %+v", stats)
|
||||
}
|
||||
got, _ := os.ReadFile(target)
|
||||
if string(got) != "EXISTING" {
|
||||
t.Fatalf("existing file should be untouched, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteExportPlanReportsFailures(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-1",
|
||||
Name: "Invoice",
|
||||
Attachments: []ItemAttachment{
|
||||
{ID: "att-1", Name: "ok.pdf", Size: 1},
|
||||
{ID: "att-2", Name: "bad.pdf", Size: 1},
|
||||
},
|
||||
}}
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
dest := t.TempDir()
|
||||
badErr := errors.New("boom")
|
||||
download := func(ctx context.Context, id string) ([]byte, error) {
|
||||
if id == "att-2" {
|
||||
return nil, badErr
|
||||
}
|
||||
return []byte("X"), nil
|
||||
}
|
||||
var out bytes.Buffer
|
||||
stats, err := executeExportPlan(context.Background(), plan, dest, download, 1, false, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no hard error from executeExportPlan, got %v", err)
|
||||
}
|
||||
if stats.written != 1 || stats.failed != 1 {
|
||||
t.Fatalf("expected 1 written, 1 failed, got %+v", stats)
|
||||
}
|
||||
if !bytes.Contains(out.Bytes(), []byte("Failed files:")) {
|
||||
t.Fatalf("expected failure summary in output, got:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportOneSetsModTime(t *testing.T) {
|
||||
mt := time.Unix(1700000000, 0)
|
||||
plan := []exportPlanEntry{{
|
||||
relPath: "Folder/2023/01/Note.pdf",
|
||||
attachmentID: "att-1",
|
||||
modTime: mt.Unix(),
|
||||
}}
|
||||
dest := t.TempDir()
|
||||
download := func(ctx context.Context, id string) ([]byte, error) { return []byte("X"), nil }
|
||||
if err := exportOne(context.Background(), plan[0], dest, download, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(dest, "Folder", "2023", "01", "Note.pdf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !info.ModTime().Equal(mt) {
|
||||
t.Fatalf("expected modtime %v, got %v", mt, info.ModTime())
|
||||
}
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// itemNode is either an itemDir or an itemFile in the pure, FUSE-independent
|
||||
// representation of the Docspell item hierarchy built by buildItemTree.
|
||||
type itemNode interface{ isItemNode() }
|
||||
|
||||
// itemDir is a directory in the pure item tree. It mirrors the FUSE dirNode but
|
||||
// carries no FUSE-specific state, so it can be reused by the mount and export
|
||||
// commands without importing bazil.org/fuse.
|
||||
type itemDir struct {
|
||||
name string
|
||||
children map[string]itemNode
|
||||
}
|
||||
|
||||
func (*itemDir) isItemNode() {}
|
||||
|
||||
// itemFile is a file entry backed by a Docspell attachment. The same *itemFile
|
||||
// pointer is referenced from both the friendly hierarchy and the by-id tree, so
|
||||
// consumers (mount) can share a single backing node between both locations.
|
||||
type itemFile struct {
|
||||
name string
|
||||
attachmentID string
|
||||
size int64
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
func (*itemFile) isItemNode() {}
|
||||
|
||||
// buildItemTree builds the pure item tree for the given Docspell search items.
|
||||
//
|
||||
// The returned root contains the friendly hierarchy (folder/year/month/<name>)
|
||||
// plus a "by-id" directory with stable ID-based lookups. Files in the friendly
|
||||
// hierarchy and the by-id tree share the same *itemFile pointers. The returned
|
||||
// metadata maps item IDs to their friendly and by-id relative paths.
|
||||
//
|
||||
// The metadata JSON file itself is intentionally not part of the tree; callers
|
||||
// that need it (the mount command) add it from the returned metadata.
|
||||
func buildItemTree(items []SearchItem) (root *itemDir, metadata mountMetadata) {
|
||||
root = &itemDir{name: "", children: map[string]itemNode{}}
|
||||
byID := &itemDir{name: "by-id", children: map[string]itemNode{}}
|
||||
root.children["by-id"] = byID
|
||||
metadata = mountMetadata{Items: map[string]mountedItemMetadata{}}
|
||||
|
||||
for _, item := range items {
|
||||
folderName := "No Folder"
|
||||
if item.Folder != nil && item.Folder.Name != "" {
|
||||
folderName = item.Folder.Name
|
||||
}
|
||||
date := item.Date
|
||||
if date == 0 {
|
||||
date = item.ItemDate
|
||||
}
|
||||
yearMonth := "No Date"
|
||||
modTime := time.Now()
|
||||
if date > 0 {
|
||||
modTime = time.UnixMilli(date)
|
||||
yearMonth = modTime.Format("2006/01")
|
||||
}
|
||||
dirParts := append([]string{folderName}, strings.Split(yearMonth, "/")...)
|
||||
dir := ensureItemDir(root, dirParts[0], dirParts[1:]...)
|
||||
|
||||
base := sanitizeName(item.Name)
|
||||
if base == "" {
|
||||
base = item.ID
|
||||
}
|
||||
|
||||
for i, att := range item.Attachments {
|
||||
ext := filepath.Ext(att.Name)
|
||||
if ext == "" {
|
||||
ext = ".pdf"
|
||||
}
|
||||
name := base + ext
|
||||
if len(item.Attachments) > 1 {
|
||||
name = fmt.Sprintf("%s - %02d%s", base, i+1, ext)
|
||||
}
|
||||
name = uniqueItemName(dir.children, sanitizeName(name))
|
||||
file := &itemFile{name: name, attachmentID: att.ID, size: att.Size, modTime: modTime}
|
||||
dir.children[name] = file
|
||||
friendlyRelPath := filepath.ToSlash(filepath.Join(append(sanitizePathParts(dirParts), name)...))
|
||||
|
||||
byIDName := item.ID + ext
|
||||
if len(item.Attachments) > 1 && i > 0 {
|
||||
byIDName = fmt.Sprintf("%s-%d%s", item.ID, i+1, ext)
|
||||
}
|
||||
shard := idShard(item.ID)
|
||||
shardDir := ensureItemDir(byID, shard)
|
||||
byIDName = uniqueItemName(shardDir.children, sanitizeName(byIDName))
|
||||
shardDir.children[byIDName] = file
|
||||
|
||||
if i == 0 {
|
||||
metadata.Items[item.ID] = mountedItemMetadata{
|
||||
FriendlyPath: friendlyRelPath,
|
||||
ByIDPath: filepath.ToSlash(filepath.Join("by-id", shard, byIDName)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return root, metadata
|
||||
}
|
||||
|
||||
func ensureItemDir(root *itemDir, first string, rest ...string) *itemDir {
|
||||
cur := root
|
||||
parts := append([]string{first}, rest...)
|
||||
for _, part := range parts {
|
||||
part = sanitizeName(part)
|
||||
if part == "" {
|
||||
part = "_"
|
||||
}
|
||||
next, ok := cur.children[part].(*itemDir)
|
||||
if !ok {
|
||||
next = &itemDir{name: part, children: map[string]itemNode{}}
|
||||
cur.children[part] = next
|
||||
}
|
||||
cur = next
|
||||
}
|
||||
return cur
|
||||
}
|
||||
|
||||
func uniqueItemName(children map[string]itemNode, name string) string {
|
||||
if _, ok := children[name]; !ok {
|
||||
return name
|
||||
}
|
||||
ext := filepath.Ext(name)
|
||||
base := strings.TrimSuffix(name, ext)
|
||||
for i := 2; ; i++ {
|
||||
candidate := fmt.Sprintf("%s (%d)%s", base, i, ext)
|
||||
if _, ok := children[candidate]; !ok {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,12 @@ func main() {
|
||||
fmt.Fprintln(os.Stderr, "mount failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
case commands.exportCommand.Used:
|
||||
cfg, password := loadConfigAndPassword()
|
||||
if err := runExportCommand(os.Args[2:], cfg, password); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "export failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
case commands.getPathCommand.Used:
|
||||
cfg, password := loadConfigAndPassword()
|
||||
if err := runGetPathCommand(os.Args[2:], cfg, password); err != nil {
|
||||
@@ -80,6 +86,7 @@ func main() {
|
||||
type cliCommands struct {
|
||||
importCommand *flaggy.Subcommand
|
||||
mountCommand *flaggy.Subcommand
|
||||
exportCommand *flaggy.Subcommand
|
||||
getPathCommand *flaggy.Subcommand
|
||||
getURLCommand *flaggy.Subcommand
|
||||
}
|
||||
@@ -93,28 +100,40 @@ func newCLIParser() (*flaggy.Parser, cliCommands) {
|
||||
commands := cliCommands{
|
||||
importCommand: flaggy.NewSubcommand("import"),
|
||||
mountCommand: flaggy.NewSubcommand("mount"),
|
||||
exportCommand: flaggy.NewSubcommand("export"),
|
||||
getPathCommand: flaggy.NewSubcommand("get-path"),
|
||||
getURLCommand: flaggy.NewSubcommand("get-url"),
|
||||
}
|
||||
commands.importCommand.Description = "Scan configured import folders, archive confirmed files, and optionally upload missing files"
|
||||
commands.mountCommand.Description = "Expose Docspell search results as a read-only FUSE filesystem"
|
||||
commands.exportCommand.Description = "Download Docspell search results into a local directory, mirroring the mount hierarchy"
|
||||
commands.getPathCommand.Description = "Resolve a Docspell item ID or URL to the local mounted path"
|
||||
commands.getURLCommand.Description = "Resolve a local mounted path, item ID, or item URL to a Docspell web URL"
|
||||
|
||||
var uploadMissing bool
|
||||
commands.importCommand.Bool(&uploadMissing, "", "upload-missing", "true, to upload files to docspell that do not yet exist there")
|
||||
var query, mountPoint, itemIDOrURL, pathOrID string
|
||||
var query, mountPoint, exportDest, itemIDOrURL, pathOrID string
|
||||
var exportConcurrency int
|
||||
var exportDryRun, exportSkipExisting bool
|
||||
authHeader := defaultDocspellAuthHeader
|
||||
limit := 1000
|
||||
commands.mountCommand.String(&query, "", "query", "Docspell item query to expose")
|
||||
commands.mountCommand.Int(&limit, "", "limit", "number of items to fetch per API request")
|
||||
commands.mountCommand.String(&authHeader, "", "auth-header", "HTTP header used for the Docspell auth token")
|
||||
commands.mountCommand.AddPositionalValue(&mountPoint, "mountpoint", 1, false, "mount point, overriding mountPath from config")
|
||||
commands.exportCommand.String(&query, "", "query", "Docspell item query to expose")
|
||||
commands.exportCommand.Int(&limit, "", "limit", "number of items to fetch per API request")
|
||||
commands.exportCommand.String(&authHeader, "", "auth-header", "HTTP header used for the Docspell auth token")
|
||||
commands.exportCommand.AddPositionalValue(&exportDest, "destination", 1, false, "destination directory for the export")
|
||||
commands.exportCommand.Int(&exportConcurrency, "", "concurrency", "number of attachments to download in parallel")
|
||||
commands.exportCommand.Bool(&exportDryRun, "", "dry-run", "print the files that would be written without downloading")
|
||||
commands.exportCommand.Bool(&exportSkipExisting, "", "skip-existing", "skip files that already exist in the destination")
|
||||
commands.getPathCommand.AddPositionalValue(&itemIDOrURL, "item-id-or-url", 1, true, "Docspell item ID or web URL")
|
||||
commands.getURLCommand.AddPositionalValue(&pathOrID, "local-path-or-item-id", 1, true, "local mounted path, item ID, or item URL")
|
||||
|
||||
parser.AttachSubcommand(commands.importCommand, 1)
|
||||
parser.AttachSubcommand(commands.mountCommand, 1)
|
||||
parser.AttachSubcommand(commands.exportCommand, 1)
|
||||
parser.AttachSubcommand(commands.getPathCommand, 1)
|
||||
parser.AttachSubcommand(commands.getURLCommand, 1)
|
||||
return parser, commands
|
||||
|
||||
@@ -337,76 +337,63 @@ func itemIDFromLocalPath(path string, cfg config) (string, error) {
|
||||
return "", fmt.Errorf("could not map mounted path to item id")
|
||||
}
|
||||
|
||||
// buildMountTree converts the pure item tree produced by buildItemTree into the
|
||||
// FUSE node tree served by the mount command. A single *fileNode is created per
|
||||
// *itemFile (shared between the friendly hierarchy and by-id lookups) so that a
|
||||
// downloaded attachment is cached once regardless of which path reads it.
|
||||
func buildMountTree(ctx context.Context, client *DocspellClient, items []SearchItem) *dirNode {
|
||||
root := &dirNode{name: "", children: map[string]fs.Node{}}
|
||||
byID := &dirNode{name: "by-id", children: map[string]fs.Node{}}
|
||||
root.children["by-id"] = byID
|
||||
metadata := mountMetadata{Items: map[string]mountedItemMetadata{}}
|
||||
metadataModTime := time.Now()
|
||||
for _, item := range items {
|
||||
folderName := "No Folder"
|
||||
if item.Folder != nil && item.Folder.Name != "" {
|
||||
folderName = item.Folder.Name
|
||||
}
|
||||
date := item.Date
|
||||
if date == 0 {
|
||||
date = item.ItemDate
|
||||
}
|
||||
yearMonth := "No Date"
|
||||
modTime := time.Now()
|
||||
if date > 0 {
|
||||
modTime = time.UnixMilli(date)
|
||||
yearMonth = modTime.Format("2006/01")
|
||||
}
|
||||
dirParts := append([]string{folderName}, strings.Split(yearMonth, "/")...)
|
||||
dir := ensureDir(root, dirParts[0], dirParts[1:]...)
|
||||
base := sanitizeName(item.Name)
|
||||
if base == "" {
|
||||
base = item.ID
|
||||
}
|
||||
for i, att := range item.Attachments {
|
||||
ext := filepath.Ext(att.Name)
|
||||
if ext == "" {
|
||||
ext = ".pdf"
|
||||
}
|
||||
name := base + ext
|
||||
if len(item.Attachments) > 1 {
|
||||
name = fmt.Sprintf("%s - %02d%s", base, i+1, ext)
|
||||
}
|
||||
size := att.Size
|
||||
if size <= 0 {
|
||||
if headSize, err := client.AttachmentSize(ctx, att.ID); err == nil && headSize > 0 {
|
||||
size = headSize
|
||||
}
|
||||
}
|
||||
name = uniqueName(dir.children, sanitizeName(name))
|
||||
node := &fileNode{name: name, attachmentID: att.ID, size: uint64(size), knownSize: size > 0, modTime: modTime, client: client}
|
||||
dir.children[name] = node
|
||||
friendlyRelPath := filepath.ToSlash(filepath.Join(append(sanitizePathParts(dirParts), name)...))
|
||||
root, metadata := buildItemTree(items)
|
||||
fuseRoot := convertItemDir(root, ctx, client, map[*itemFile]*fileNode{})
|
||||
|
||||
byIDName := item.ID + ext
|
||||
if len(item.Attachments) > 1 {
|
||||
if i > 0 {
|
||||
byIDName = fmt.Sprintf("%s-%d%s", item.ID, i+1, ext)
|
||||
}
|
||||
}
|
||||
shard := idShard(item.ID)
|
||||
shardDir := ensureDir(byID, shard)
|
||||
byIDName = uniqueName(shardDir.children, sanitizeName(byIDName))
|
||||
shardDir.children[byIDName] = node
|
||||
if i == 0 {
|
||||
metadata.Items[item.ID] = mountedItemMetadata{
|
||||
FriendlyPath: friendlyRelPath,
|
||||
ByIDPath: filepath.ToSlash(filepath.Join("by-id", shard, byIDName)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
metadataData, err := json.MarshalIndent(metadata, "", " ")
|
||||
if err == nil {
|
||||
root.children[mountMetadataFileName] = &staticFileNode{name: mountMetadataFileName, data: append(metadataData, '\n'), modTime: metadataModTime}
|
||||
fuseRoot.children[mountMetadataFileName] = &staticFileNode{
|
||||
name: mountMetadataFileName,
|
||||
data: append(metadataData, '\n'),
|
||||
modTime: time.Now(),
|
||||
}
|
||||
return root
|
||||
}
|
||||
return fuseRoot
|
||||
}
|
||||
|
||||
// convertItemDir recursively converts a pure itemDir into a FUSE dirNode. The
|
||||
// fileCache ensures that the same *itemFile (referenced from both the friendly
|
||||
// hierarchy and the by-id tree) maps to a single shared *fileNode.
|
||||
func convertItemDir(dir *itemDir, ctx context.Context, client *DocspellClient, fileCache map[*itemFile]*fileNode) *dirNode {
|
||||
node := &dirNode{name: dir.name, children: map[string]fs.Node{}}
|
||||
for name, child := range dir.children {
|
||||
switch c := child.(type) {
|
||||
case *itemDir:
|
||||
node.children[name] = convertItemDir(c, ctx, client, fileCache)
|
||||
case *itemFile:
|
||||
node.children[name] = convertItemFile(c, ctx, client, fileCache)
|
||||
}
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
func convertItemFile(file *itemFile, ctx context.Context, client *DocspellClient, fileCache map[*itemFile]*fileNode) *fileNode {
|
||||
if cached, ok := fileCache[file]; ok {
|
||||
return cached
|
||||
}
|
||||
size := file.size
|
||||
knownSize := size > 0
|
||||
if size <= 0 {
|
||||
if headSize, err := client.AttachmentSize(ctx, file.attachmentID); err == nil && headSize > 0 {
|
||||
size = headSize
|
||||
knownSize = true
|
||||
}
|
||||
}
|
||||
node := &fileNode{
|
||||
name: file.name,
|
||||
attachmentID: file.attachmentID,
|
||||
size: uint64(size),
|
||||
knownSize: knownSize,
|
||||
modTime: file.modTime,
|
||||
client: client,
|
||||
}
|
||||
fileCache[file] = node
|
||||
return node
|
||||
}
|
||||
|
||||
func sanitizePathParts(parts []string) []string {
|
||||
@@ -432,24 +419,6 @@ func idShard(id string) string {
|
||||
return "__"
|
||||
}
|
||||
|
||||
func ensureDir(root *dirNode, first string, rest ...string) *dirNode {
|
||||
cur := root
|
||||
parts := append([]string{first}, rest...)
|
||||
for _, part := range parts {
|
||||
part = sanitizeName(part)
|
||||
if part == "" {
|
||||
part = "_"
|
||||
}
|
||||
next, ok := cur.children[part].(*dirNode)
|
||||
if !ok {
|
||||
next = &dirNode{name: part, children: map[string]fs.Node{}}
|
||||
cur.children[part] = next
|
||||
}
|
||||
cur = next
|
||||
}
|
||||
return cur
|
||||
}
|
||||
|
||||
func sanitizeName(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
s = strings.ReplaceAll(s, "/", "_")
|
||||
@@ -457,20 +426,6 @@ func sanitizeName(s string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
func uniqueName(children map[string]fs.Node, name string) string {
|
||||
if _, ok := children[name]; !ok {
|
||||
return name
|
||||
}
|
||||
ext := filepath.Ext(name)
|
||||
base := strings.TrimSuffix(name, ext)
|
||||
for i := 2; ; i++ {
|
||||
candidate := fmt.Sprintf("%s (%d)%s", base, i, ext)
|
||||
if _, ok := children[candidate]; !ok {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *docspellFS) Root() (fs.Node, error) { return f.root, nil }
|
||||
|
||||
func (d *dirNode) Attr(ctx context.Context, a *fuse.Attr) error {
|
||||
|
||||
Reference in new issue
Block a user