Add export command and shared item tree
Introduces a pure, FUSE-independent item tree (buildItemTree) shared by the mount and export commands. The new export subcommand downloads Docspell search results into a local directory, mirroring the mount's friendly folder/year/month hierarchy without the by-id directory or metadata JSON file, for archival or migration. - itemtree.go: pure item tree builder shared by mount and export - mount.go: buildMountTree now converts the pure tree into FUSE nodes, with a shared fileNode per attachment across friendly and by-id paths - export.go: export command with concurrent downloads, --dry-run, --skip-existing, --concurrency, --query, --limit, --auth-header - main.go: wires up the export subcommand and its flags - export_test.go: tests for plan collection, writes, skip-existing, failures, modtime - README.md: documents the export command
This commit is contained in:
Jan Bader committed
1 parent
c4a2e9d085
commit
5e88eaea97
6 files changed
+696
-98
No files matched your search
+214
@@ -0,0 +1,214 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCollectExportPlanMirrorsFriendlyHierarchy(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-1",
|
||||
Name: "Invoice/July",
|
||||
Date: 1720396800000, // 2024-07-08 UTC
|
||||
Folder: &DocspellEntity{
|
||||
Name: "Bills/Private",
|
||||
},
|
||||
Attachments: []ItemAttachment{
|
||||
{ID: "att-1", Name: "invoice.pdf", Size: 123},
|
||||
{ID: "att-2", Name: "scan.pdf", Size: 456},
|
||||
},
|
||||
}}
|
||||
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
want := []string{
|
||||
"Bills_Private/2024/07/Invoice_July - 01.pdf",
|
||||
"Bills_Private/2024/07/Invoice_July - 02.pdf",
|
||||
}
|
||||
if len(plan) != len(want) {
|
||||
t.Fatalf("expected %d entries, got %d: %#v", len(want), len(plan), plan)
|
||||
}
|
||||
for i, e := range plan {
|
||||
if e.relPath != want[i] {
|
||||
t.Errorf("entry %d: expected %q, got %q", i, want[i], e.relPath)
|
||||
}
|
||||
}
|
||||
if plan[0].attachmentID != "att-1" || plan[1].attachmentID != "att-2" {
|
||||
t.Errorf("attachment ids not preserved: %+v", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectExportPlanSkipsByIDAndMetadata(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-1",
|
||||
Name: "Invoice",
|
||||
Attachments: []ItemAttachment{{ID: "att-1", Name: "invoice.pdf", Size: 1}},
|
||||
}}
|
||||
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
if len(plan) != 1 {
|
||||
t.Fatalf("expected a single friendly entry, got %d: %#v", len(plan), plan)
|
||||
}
|
||||
for _, e := range plan {
|
||||
if strings.HasPrefix(e.relPath, "by-id/") {
|
||||
t.Errorf("plan must not include by-id entries, got %q", e.relPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectExportPlanItemsWithoutFolderOrDate(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-2",
|
||||
Name: "Undated Note",
|
||||
Attachments: []ItemAttachment{{ID: "att-3", Name: "note", Size: 1}},
|
||||
}}
|
||||
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
if len(plan) != 1 || plan[0].relPath != "No Folder/No Date/Undated Note.pdf" {
|
||||
t.Fatalf("unexpected plan: %#v", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteExportPlanWritesFilesAndContent(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-1",
|
||||
Name: "Invoice",
|
||||
Date: 1720396800000,
|
||||
Folder: &DocspellEntity{Name: "Bills"},
|
||||
Attachments: []ItemAttachment{
|
||||
{ID: "att-1", Name: "invoice.pdf", Size: 4},
|
||||
{ID: "att-2", Name: "scan.pdf", Size: 4},
|
||||
},
|
||||
}}
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
dest := t.TempDir()
|
||||
contents := map[string]string{"att-1": "AAAA", "att-2": "BBBB"}
|
||||
download := func(ctx context.Context, id string) ([]byte, error) {
|
||||
return []byte(contents[id]), nil
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
stats, err := executeExportPlan(context.Background(), plan, dest, download, 2, false, &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stats.written != 2 || stats.skipped != 0 || stats.failed != 0 {
|
||||
t.Fatalf("unexpected stats: %+v", stats)
|
||||
}
|
||||
|
||||
for rel, content := range map[string]string{
|
||||
"Bills/2024/07/Invoice - 01.pdf": "AAAA",
|
||||
"Bills/2024/07/Invoice - 02.pdf": "BBBB",
|
||||
} {
|
||||
got, err := os.ReadFile(filepath.Join(dest, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", rel, err)
|
||||
}
|
||||
if string(got) != content {
|
||||
t.Errorf("content for %s: expected %q, got %q", rel, content, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteExportPlanSkipExisting(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-1",
|
||||
Name: "Invoice",
|
||||
Attachments: []ItemAttachment{{ID: "att-1", Name: "invoice.pdf", Size: 4}},
|
||||
}}
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
dest := t.TempDir()
|
||||
target := filepath.Join(dest, "No Folder", "No Date", "Invoice.pdf")
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(target, []byte("EXISTING"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
download := func(ctx context.Context, id string) ([]byte, error) {
|
||||
t.Fatal("download should not be called for skipped existing file")
|
||||
return nil, nil
|
||||
}
|
||||
var out bytes.Buffer
|
||||
stats, err := executeExportPlan(context.Background(), plan, dest, download, 1, true, &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stats.skipped != 1 || stats.written != 0 {
|
||||
t.Fatalf("expected 1 skipped, 0 written, got %+v", stats)
|
||||
}
|
||||
got, _ := os.ReadFile(target)
|
||||
if string(got) != "EXISTING" {
|
||||
t.Fatalf("existing file should be untouched, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteExportPlanReportsFailures(t *testing.T) {
|
||||
items := []SearchItem{{
|
||||
ID: "item-1",
|
||||
Name: "Invoice",
|
||||
Attachments: []ItemAttachment{
|
||||
{ID: "att-1", Name: "ok.pdf", Size: 1},
|
||||
{ID: "att-2", Name: "bad.pdf", Size: 1},
|
||||
},
|
||||
}}
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
dest := t.TempDir()
|
||||
badErr := errors.New("boom")
|
||||
download := func(ctx context.Context, id string) ([]byte, error) {
|
||||
if id == "att-2" {
|
||||
return nil, badErr
|
||||
}
|
||||
return []byte("X"), nil
|
||||
}
|
||||
var out bytes.Buffer
|
||||
stats, err := executeExportPlan(context.Background(), plan, dest, download, 1, false, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no hard error from executeExportPlan, got %v", err)
|
||||
}
|
||||
if stats.written != 1 || stats.failed != 1 {
|
||||
t.Fatalf("expected 1 written, 1 failed, got %+v", stats)
|
||||
}
|
||||
if !bytes.Contains(out.Bytes(), []byte("Failed files:")) {
|
||||
t.Fatalf("expected failure summary in output, got:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportOneSetsModTime(t *testing.T) {
|
||||
mt := time.Unix(1700000000, 0)
|
||||
plan := []exportPlanEntry{{
|
||||
relPath: "Folder/2023/01/Note.pdf",
|
||||
attachmentID: "att-1",
|
||||
modTime: mt.Unix(),
|
||||
}}
|
||||
dest := t.TempDir()
|
||||
download := func(ctx context.Context, id string) ([]byte, error) { return []byte("X"), nil }
|
||||
if err := exportOne(context.Background(), plan[0], dest, download, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(dest, "Folder", "2023", "01", "Note.pdf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !info.ModTime().Equal(mt) {
|
||||
t.Fatalf("expected modtime %v, got %v", mt, info.ModTime())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user