docspell-cli/export.go

package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"sync/atomic"
	"time"

	"github.com/integrii/flaggy"
)

// byIDDirName is the directory name used by buildItemTree for the ID-based
// lookup tree. The export command mirrors only the friendly hierarchy, so it
// skips this directory.
const byIDDirName = "by-id"

// runExportCommand downloads Docspell search results into a local directory,
// reproducing the same folder/year/month/<name> hierarchy exposed by the mount
// command but without the by-id directory or the metadata JSON file. This is
// useful for archival or migration to a plain on-disk layout.
func runExportCommand(args []string, cfg config, password string) error {
	options, err := parseExportOptions(args, cfg)
	if err != nil {
		return err
	}
	if options.destination == "" {
		return fmt.Errorf("export destination missing: pass a destination directory as the first positional argument")
	}

	absDest, err := filepath.Abs(options.destination)
	if err != nil {
		return fmt.Errorf("resolve destination: %w", err)
	}
	if info, err := os.Stat(absDest); err == nil && !info.IsDir() {
		return fmt.Errorf("destination %s exists and is not a directory", absDest)
	}

	ctx := context.Background()
	client := NewDocspellClient(cfg.DocspellURL, options.authHeader)
	if err := client.Login(ctx, cfg.User, password); err != nil {
		return fmt.Errorf("login: %w", err)
	}

	fmt.Printf("Searching Docspell items (query=%q, limit=%d)...\n", options.query, options.limit)
	items, err := client.SearchItems(ctx, options.query, options.limit)
	if err != nil {
		return fmt.Errorf("search items: %w", err)
	}
	root, _ := buildItemTree(items)
	plan := collectExportPlan(root)

	if options.dryRun {
		fmt.Printf("Dry run: %d file(s) would be written under %s\n", len(plan), absDest)
		for _, entry := range plan {
			fmt.Println("  ", filepath.FromSlash(entry.relPath))
		}
		return nil
	}

	fmt.Printf("Exporting %d file(s) to %s (concurrency=%d)...\n", len(plan), absDest, options.concurrency)
	if err := os.MkdirAll(absDest, 0755); err != nil {
		return fmt.Errorf("create destination: %w", err)
	}
	download := func(ctx context.Context, attachmentID string) ([]byte, error) {
		return client.DownloadAttachment(ctx, attachmentID)
	}
	stats, err := executeExportPlan(ctx, plan, absDest, download, options.concurrency, options.skipExisting, os.Stdout)
	if err != nil {
		return err
	}
	fmt.Printf("Export complete: %d written, %d skipped, %d failed out of %d total.\n",
		stats.written, stats.skipped, stats.failed, len(plan))
	if stats.failed > 0 {
		return fmt.Errorf("%d file(s) failed to export", stats.failed)
	}
	return nil
}

type exportOptions struct {
	query       string
	limit       int
	authHeader  string
	destination string
	concurrency int
	dryRun      bool
	skipExisting bool
}

func parseExportOptions(args []string, cfg config) (exportOptions, error) {
	options := exportOptions{
		query:        cfg.MountQuery,
		limit:        1000,
		authHeader:   defaultDocspellAuthHeader,
		concurrency:  4,
	}
	parser := flaggy.NewParser("export")
	parser.Description = "Download Docspell search results into a local directory, mirroring the mount hierarchy"
	parser.DisableShowVersionWithVersion()
	parser.ShowHelpOnUnexpected = true
	parser.String(&options.query, "", "query", "Docspell item query to expose")
	parser.Int(&options.limit, "", "limit", "number of items to fetch per API request")
	parser.String(&options.authHeader, "", "auth-header", "HTTP header used for the Docspell auth token")
	parser.Int(&options.concurrency, "", "concurrency", "number of attachments to download in parallel")
	parser.Bool(&options.dryRun, "", "dry-run", "print the files that would be written without downloading")
	parser.Bool(&options.skipExisting, "", "skip-existing", "skip files that already exist in the destination")
	parser.AddPositionalValue(&options.destination, "destination", 1, false, "destination directory for the export")
	return options, parser.ParseArgs(args)
}

// exportPlanEntry describes a single file to write into the export destination.
// relPath is slash-separated and relative to the destination root.
type exportPlanEntry struct {
	relPath      string
	attachmentID string
	modTime      int64 // unix seconds; 0 means preserve current time
}

// collectExportPlan walks the friendly hierarchy of the item tree (skipping the
// by-id directory and any metadata file) and returns the files to write in a
// deterministic, depth-first, name-sorted order.
func collectExportPlan(root *itemDir) []exportPlanEntry {
	var plan []exportPlanEntry
	for name, child := range root.children {
		if name == byIDDirName || name == mountMetadataFileName {
			continue
		}
		collectExportPlanNode(child, name, &plan)
	}
	return plan
}

func collectExportPlanNode(node itemNode, relPath string, plan *[]exportPlanEntry) {
	switch n := node.(type) {
	case *itemDir:
		names := make([]string, 0, len(n.children))
		for name := range n.children {
			names = append(names, name)
		}
		sort.Strings(names)
		for _, name := range names {
			collectExportPlanNode(n.children[name], relPath+"/"+name, plan)
		}
	case *itemFile:
		modTime := int64(0)
		if !n.modTime.IsZero() {
			modTime = n.modTime.Unix()
		}
		*plan = append(*plan, exportPlanEntry{
			relPath:      relPath,
			attachmentID: n.attachmentID,
			modTime:      modTime,
		})
	}
}

type exportStats struct {
	written int64
	skipped int64
	failed  int64
}

func (s *exportStats) addWritten() { atomic.AddInt64(&s.written, 1) }
func (s *exportStats) addSkipped() { atomic.AddInt64(&s.skipped, 1) }
func (s *exportStats) addFailed()  { atomic.AddInt64(&s.failed, 1) }

// downloadFunc fetches an attachment by ID. It is a parameter so tests can
// substitute an in-memory implementation.
type downloadFunc func(ctx context.Context, attachmentID string) ([]byte, error)

// executeExportPlan downloads and writes every entry in plan under dest. Up to
// concurrency attachments are fetched in parallel. Errors from individual
// attachments are reported but do not abort the whole run.
func executeExportPlan(ctx context.Context, plan []exportPlanEntry, dest string, download downloadFunc, concurrency int, skipExisting bool, out io.Writer) (exportStats, error) {
	if concurrency < 1 {
		concurrency = 1
	}

	jobs := make(chan exportPlanEntry)
	var stats exportStats
	var reasonsMu sync.Mutex
	var failedReasons []string

	var wg sync.WaitGroup
	for w := 0; w < concurrency; w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for entry := range jobs {
				switch err := exportOne(ctx, entry, dest, download, skipExisting); {
				case errors.Is(err, errSkipExisting):
					stats.addSkipped()
					fmt.Fprintln(out, "  skip", filepath.FromSlash(entry.relPath))
				case err != nil:
					stats.addFailed()
					reasonsMu.Lock()
					failedReasons = append(failedReasons, fmt.Sprintf("%s: %v", filepath.FromSlash(entry.relPath), err))
					reasonsMu.Unlock()
					fmt.Fprintln(out, "  FAIL", filepath.FromSlash(entry.relPath), "-", err)
				default:
					stats.addWritten()
					fmt.Fprintln(out, "  wrote", filepath.FromSlash(entry.relPath))
				}
			}
		}()
	}

	for _, entry := range plan {
		jobs <- entry
	}
	close(jobs)
	wg.Wait()

	if len(failedReasons) > 0 {
		fmt.Fprintln(out, "Failed files:")
		for _, reason := range failedReasons {
			fmt.Fprintln(out, "  -", reason)
		}
	}
	return stats, nil
}

var errSkipExisting = errors.New("skip existing file")

func exportOne(ctx context.Context, entry exportPlanEntry, dest string, download downloadFunc, skipExisting bool) error {
	target := filepath.Join(dest, filepath.FromSlash(entry.relPath))
	if skipExisting {
		if _, err := os.Stat(target); err == nil {
			return errSkipExisting
		}
	}
	data, err := download(ctx, entry.attachmentID)
	if err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
		return err
	}
	if err := os.WriteFile(target, data, 0644); err != nil {
		return err
	}
	if entry.modTime != 0 {
		t := time.Unix(entry.modTime, 0)
		_ = os.Chtimes(target, t, t)
	}
	return nil
}