Add export command and shared item tree
Introduces a pure, FUSE-independent item tree (buildItemTree) shared by the mount and export commands. The new export subcommand downloads Docspell search results into a local directory, mirroring the mount's friendly folder/year/month hierarchy without the by-id directory or metadata JSON file, for archival or migration. - itemtree.go: pure item tree builder shared by mount and export - mount.go: buildMountTree now converts the pure tree into FUSE nodes, with a shared fileNode per attachment across friendly and by-id paths - export.go: export command with concurrent downloads, --dry-run, --skip-existing, --concurrency, --query, --limit, --auth-header - main.go: wires up the export subcommand and its flags - export_test.go: tests for plan collection, writes, skip-existing, failures, modtime - README.md: documents the export command
This commit is contained in:
Jan Bader committed
1 parent
c4a2e9d085
commit
5e88eaea97
6 files changed
+696
-98
No files matched your search
@@ -0,0 +1,252 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/integrii/flaggy"
|
||||
)
|
||||
|
||||
// byIDDirName is the directory name used by buildItemTree for the ID-based
|
||||
// lookup tree. The export command mirrors only the friendly hierarchy, so it
|
||||
// skips this directory.
|
||||
const byIDDirName = "by-id"
|
||||
|
||||
// runExportCommand downloads Docspell search results into a local directory,
|
||||
// reproducing the same folder/year/month/<name> hierarchy exposed by the mount
|
||||
// command but without the by-id directory or the metadata JSON file. This is
|
||||
// useful for archival or migration to a plain on-disk layout.
|
||||
func runExportCommand(args []string, cfg config, password string) error {
|
||||
options, err := parseExportOptions(args, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if options.destination == "" {
|
||||
return fmt.Errorf("export destination missing: pass a destination directory as the first positional argument")
|
||||
}
|
||||
|
||||
absDest, err := filepath.Abs(options.destination)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve destination: %w", err)
|
||||
}
|
||||
if info, err := os.Stat(absDest); err == nil && !info.IsDir() {
|
||||
return fmt.Errorf("destination %s exists and is not a directory", absDest)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
client := NewDocspellClient(cfg.DocspellURL, options.authHeader)
|
||||
if err := client.Login(ctx, cfg.User, password); err != nil {
|
||||
return fmt.Errorf("login: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Searching Docspell items (query=%q, limit=%d)...\n", options.query, options.limit)
|
||||
items, err := client.SearchItems(ctx, options.query, options.limit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("search items: %w", err)
|
||||
}
|
||||
root, _ := buildItemTree(items)
|
||||
plan := collectExportPlan(root)
|
||||
|
||||
if options.dryRun {
|
||||
fmt.Printf("Dry run: %d file(s) would be written under %s\n", len(plan), absDest)
|
||||
for _, entry := range plan {
|
||||
fmt.Println(" ", filepath.FromSlash(entry.relPath))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("Exporting %d file(s) to %s (concurrency=%d)...\n", len(plan), absDest, options.concurrency)
|
||||
if err := os.MkdirAll(absDest, 0755); err != nil {
|
||||
return fmt.Errorf("create destination: %w", err)
|
||||
}
|
||||
download := func(ctx context.Context, attachmentID string) ([]byte, error) {
|
||||
return client.DownloadAttachment(ctx, attachmentID)
|
||||
}
|
||||
stats, err := executeExportPlan(ctx, plan, absDest, download, options.concurrency, options.skipExisting, os.Stdout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Export complete: %d written, %d skipped, %d failed out of %d total.\n",
|
||||
stats.written, stats.skipped, stats.failed, len(plan))
|
||||
if stats.failed > 0 {
|
||||
return fmt.Errorf("%d file(s) failed to export", stats.failed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type exportOptions struct {
|
||||
query string
|
||||
limit int
|
||||
authHeader string
|
||||
destination string
|
||||
concurrency int
|
||||
dryRun bool
|
||||
skipExisting bool
|
||||
}
|
||||
|
||||
func parseExportOptions(args []string, cfg config) (exportOptions, error) {
|
||||
options := exportOptions{
|
||||
query: cfg.MountQuery,
|
||||
limit: 1000,
|
||||
authHeader: defaultDocspellAuthHeader,
|
||||
concurrency: 4,
|
||||
}
|
||||
parser := flaggy.NewParser("export")
|
||||
parser.Description = "Download Docspell search results into a local directory, mirroring the mount hierarchy"
|
||||
parser.DisableShowVersionWithVersion()
|
||||
parser.ShowHelpOnUnexpected = true
|
||||
parser.String(&options.query, "", "query", "Docspell item query to expose")
|
||||
parser.Int(&options.limit, "", "limit", "number of items to fetch per API request")
|
||||
parser.String(&options.authHeader, "", "auth-header", "HTTP header used for the Docspell auth token")
|
||||
parser.Int(&options.concurrency, "", "concurrency", "number of attachments to download in parallel")
|
||||
parser.Bool(&options.dryRun, "", "dry-run", "print the files that would be written without downloading")
|
||||
parser.Bool(&options.skipExisting, "", "skip-existing", "skip files that already exist in the destination")
|
||||
parser.AddPositionalValue(&options.destination, "destination", 1, false, "destination directory for the export")
|
||||
return options, parser.ParseArgs(args)
|
||||
}
|
||||
|
||||
// exportPlanEntry describes a single file to write into the export destination.
|
||||
// relPath is slash-separated and relative to the destination root.
|
||||
type exportPlanEntry struct {
|
||||
relPath string
|
||||
attachmentID string
|
||||
modTime int64 // unix seconds; 0 means preserve current time
|
||||
}
|
||||
|
||||
// collectExportPlan walks the friendly hierarchy of the item tree (skipping the
|
||||
// by-id directory and any metadata file) and returns the files to write in a
|
||||
// deterministic, depth-first, name-sorted order.
|
||||
func collectExportPlan(root *itemDir) []exportPlanEntry {
|
||||
var plan []exportPlanEntry
|
||||
for name, child := range root.children {
|
||||
if name == byIDDirName || name == mountMetadataFileName {
|
||||
continue
|
||||
}
|
||||
collectExportPlanNode(child, name, &plan)
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
func collectExportPlanNode(node itemNode, relPath string, plan *[]exportPlanEntry) {
|
||||
switch n := node.(type) {
|
||||
case *itemDir:
|
||||
names := make([]string, 0, len(n.children))
|
||||
for name := range n.children {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
collectExportPlanNode(n.children[name], relPath+"/"+name, plan)
|
||||
}
|
||||
case *itemFile:
|
||||
modTime := int64(0)
|
||||
if !n.modTime.IsZero() {
|
||||
modTime = n.modTime.Unix()
|
||||
}
|
||||
*plan = append(*plan, exportPlanEntry{
|
||||
relPath: relPath,
|
||||
attachmentID: n.attachmentID,
|
||||
modTime: modTime,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type exportStats struct {
|
||||
written int64
|
||||
skipped int64
|
||||
failed int64
|
||||
}
|
||||
|
||||
func (s *exportStats) addWritten() { atomic.AddInt64(&s.written, 1) }
|
||||
func (s *exportStats) addSkipped() { atomic.AddInt64(&s.skipped, 1) }
|
||||
func (s *exportStats) addFailed() { atomic.AddInt64(&s.failed, 1) }
|
||||
|
||||
// downloadFunc fetches an attachment by ID. It is a parameter so tests can
|
||||
// substitute an in-memory implementation.
|
||||
type downloadFunc func(ctx context.Context, attachmentID string) ([]byte, error)
|
||||
|
||||
// executeExportPlan downloads and writes every entry in plan under dest. Up to
|
||||
// concurrency attachments are fetched in parallel. Errors from individual
|
||||
// attachments are reported but do not abort the whole run.
|
||||
func executeExportPlan(ctx context.Context, plan []exportPlanEntry, dest string, download downloadFunc, concurrency int, skipExisting bool, out io.Writer) (exportStats, error) {
|
||||
if concurrency < 1 {
|
||||
concurrency = 1
|
||||
}
|
||||
|
||||
jobs := make(chan exportPlanEntry)
|
||||
var stats exportStats
|
||||
var reasonsMu sync.Mutex
|
||||
var failedReasons []string
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < concurrency; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for entry := range jobs {
|
||||
switch err := exportOne(ctx, entry, dest, download, skipExisting); {
|
||||
case errors.Is(err, errSkipExisting):
|
||||
stats.addSkipped()
|
||||
fmt.Fprintln(out, " skip", filepath.FromSlash(entry.relPath))
|
||||
case err != nil:
|
||||
stats.addFailed()
|
||||
reasonsMu.Lock()
|
||||
failedReasons = append(failedReasons, fmt.Sprintf("%s: %v", filepath.FromSlash(entry.relPath), err))
|
||||
reasonsMu.Unlock()
|
||||
fmt.Fprintln(out, " FAIL", filepath.FromSlash(entry.relPath), "-", err)
|
||||
default:
|
||||
stats.addWritten()
|
||||
fmt.Fprintln(out, " wrote", filepath.FromSlash(entry.relPath))
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for _, entry := range plan {
|
||||
jobs <- entry
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
|
||||
if len(failedReasons) > 0 {
|
||||
fmt.Fprintln(out, "Failed files:")
|
||||
for _, reason := range failedReasons {
|
||||
fmt.Fprintln(out, " -", reason)
|
||||
}
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
var errSkipExisting = errors.New("skip existing file")
|
||||
|
||||
func exportOne(ctx context.Context, entry exportPlanEntry, dest string, download downloadFunc, skipExisting bool) error {
|
||||
target := filepath.Join(dest, filepath.FromSlash(entry.relPath))
|
||||
if skipExisting {
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
return errSkipExisting
|
||||
}
|
||||
}
|
||||
data, err := download(ctx, entry.attachmentID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(target, data, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.modTime != 0 {
|
||||
t := time.Unix(entry.modTime, 0)
|
||||
_ = os.Chtimes(target, t, t)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user