package main import ( "context" "errors" "fmt" "io" "os" "path/filepath" "sort" "sync" "sync/atomic" "time" "github.com/integrii/flaggy" ) // byIDDirName is the directory name used by buildItemTree for the ID-based // lookup tree. The export command mirrors only the friendly hierarchy, so it // skips this directory. const byIDDirName = "by-id" // runExportCommand downloads Docspell search results into a local directory, // reproducing the same folder/year/month/ hierarchy exposed by the mount // command but without the by-id directory or the metadata JSON file. This is // useful for archival or migration to a plain on-disk layout. func runExportCommand(args []string, cfg config, password string) error { options, err := parseExportOptions(args, cfg) if err != nil { return err } if options.destination == "" { return fmt.Errorf("export destination missing: pass a destination directory as the first positional argument") } absDest, err := filepath.Abs(options.destination) if err != nil { return fmt.Errorf("resolve destination: %w", err) } if info, err := os.Stat(absDest); err == nil && !info.IsDir() { return fmt.Errorf("destination %s exists and is not a directory", absDest) } ctx := context.Background() client := NewDocspellClient(cfg.DocspellURL, options.authHeader) if err := client.Login(ctx, cfg.User, password); err != nil { return fmt.Errorf("login: %w", err) } fmt.Printf("Searching Docspell items (query=%q, limit=%d)...\n", options.query, options.limit) items, err := client.SearchItems(ctx, options.query, options.limit) if err != nil { return fmt.Errorf("search items: %w", err) } root, _ := buildItemTree(items) plan := collectExportPlan(root) if options.dryRun { fmt.Printf("Dry run: %d file(s) would be written under %s\n", len(plan), absDest) for _, entry := range plan { fmt.Println(" ", filepath.FromSlash(entry.relPath)) } return nil } fmt.Printf("Exporting %d file(s) to %s (concurrency=%d)...\n", len(plan), absDest, options.concurrency) if err := os.MkdirAll(absDest, 0755); err != nil { return fmt.Errorf("create destination: %w", err) } download := func(ctx context.Context, attachmentID string) ([]byte, error) { return client.DownloadAttachment(ctx, attachmentID) } stats, err := executeExportPlan(ctx, plan, absDest, download, options.concurrency, options.skipExisting, os.Stdout) if err != nil { return err } fmt.Printf("Export complete: %d written, %d skipped, %d failed out of %d total.\n", stats.written, stats.skipped, stats.failed, len(plan)) if stats.failed > 0 { return fmt.Errorf("%d file(s) failed to export", stats.failed) } return nil } type exportOptions struct { query string limit int authHeader string destination string concurrency int dryRun bool skipExisting bool } func parseExportOptions(args []string, cfg config) (exportOptions, error) { options := exportOptions{ query: cfg.MountQuery, limit: 1000, authHeader: defaultDocspellAuthHeader, concurrency: 4, } parser := flaggy.NewParser("export") parser.Description = "Download Docspell search results into a local directory, mirroring the mount hierarchy" parser.DisableShowVersionWithVersion() parser.ShowHelpOnUnexpected = true parser.String(&options.query, "", "query", "Docspell item query to expose") parser.Int(&options.limit, "", "limit", "number of items to fetch per API request") parser.String(&options.authHeader, "", "auth-header", "HTTP header used for the Docspell auth token") parser.Int(&options.concurrency, "", "concurrency", "number of attachments to download in parallel") parser.Bool(&options.dryRun, "", "dry-run", "print the files that would be written without downloading") parser.Bool(&options.skipExisting, "", "skip-existing", "skip files that already exist in the destination") parser.AddPositionalValue(&options.destination, "destination", 1, false, "destination directory for the export") return options, parser.ParseArgs(args) } // exportPlanEntry describes a single file to write into the export destination. // relPath is slash-separated and relative to the destination root. type exportPlanEntry struct { relPath string attachmentID string modTime int64 // unix seconds; 0 means preserve current time } // collectExportPlan walks the friendly hierarchy of the item tree (skipping the // by-id directory and any metadata file) and returns the files to write in a // deterministic, depth-first, name-sorted order. func collectExportPlan(root *itemDir) []exportPlanEntry { var plan []exportPlanEntry for name, child := range root.children { if name == byIDDirName || name == mountMetadataFileName { continue } collectExportPlanNode(child, name, &plan) } return plan } func collectExportPlanNode(node itemNode, relPath string, plan *[]exportPlanEntry) { switch n := node.(type) { case *itemDir: names := make([]string, 0, len(n.children)) for name := range n.children { names = append(names, name) } sort.Strings(names) for _, name := range names { collectExportPlanNode(n.children[name], relPath+"/"+name, plan) } case *itemFile: modTime := int64(0) if !n.modTime.IsZero() { modTime = n.modTime.Unix() } *plan = append(*plan, exportPlanEntry{ relPath: relPath, attachmentID: n.attachmentID, modTime: modTime, }) } } type exportStats struct { written int64 skipped int64 failed int64 } func (s *exportStats) addWritten() { atomic.AddInt64(&s.written, 1) } func (s *exportStats) addSkipped() { atomic.AddInt64(&s.skipped, 1) } func (s *exportStats) addFailed() { atomic.AddInt64(&s.failed, 1) } // downloadFunc fetches an attachment by ID. It is a parameter so tests can // substitute an in-memory implementation. type downloadFunc func(ctx context.Context, attachmentID string) ([]byte, error) // executeExportPlan downloads and writes every entry in plan under dest. Up to // concurrency attachments are fetched in parallel. Errors from individual // attachments are reported but do not abort the whole run. func executeExportPlan(ctx context.Context, plan []exportPlanEntry, dest string, download downloadFunc, concurrency int, skipExisting bool, out io.Writer) (exportStats, error) { if concurrency < 1 { concurrency = 1 } jobs := make(chan exportPlanEntry) var stats exportStats var reasonsMu sync.Mutex var failedReasons []string var wg sync.WaitGroup for w := 0; w < concurrency; w++ { wg.Add(1) go func() { defer wg.Done() for entry := range jobs { switch err := exportOne(ctx, entry, dest, download, skipExisting); { case errors.Is(err, errSkipExisting): stats.addSkipped() fmt.Fprintln(out, " skip", filepath.FromSlash(entry.relPath)) case err != nil: stats.addFailed() reasonsMu.Lock() failedReasons = append(failedReasons, fmt.Sprintf("%s: %v", filepath.FromSlash(entry.relPath), err)) reasonsMu.Unlock() fmt.Fprintln(out, " FAIL", filepath.FromSlash(entry.relPath), "-", err) default: stats.addWritten() fmt.Fprintln(out, " wrote", filepath.FromSlash(entry.relPath)) } } }() } for _, entry := range plan { jobs <- entry } close(jobs) wg.Wait() if len(failedReasons) > 0 { fmt.Fprintln(out, "Failed files:") for _, reason := range failedReasons { fmt.Fprintln(out, " -", reason) } } return stats, nil } var errSkipExisting = errors.New("skip existing file") func exportOne(ctx context.Context, entry exportPlanEntry, dest string, download downloadFunc, skipExisting bool) error { target := filepath.Join(dest, filepath.FromSlash(entry.relPath)) if skipExisting { if _, err := os.Stat(target); err == nil { return errSkipExisting } } data, err := download(ctx, entry.attachmentID) if err != nil { return err } if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { return err } if err := os.WriteFile(target, data, 0644); err != nil { return err } if entry.modTime != 0 { t := time.Unix(entry.modTime, 0) _ = os.Chtimes(target, t, t) } return nil }