From 3b91a26b2c310eac99b838aaf453fabc1078a33b Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Sat, 19 Sep 2026 23:36:10 +0200 Subject: [PATCH] feat: mount lazyly --- README.md | 10 +- mount.go | 261 ++++++++++++++++++++++++++++++++++++++++++++++++-- mount_test.go | 122 +++++++++++++++++++++++ 3 files changed, 381 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 5e43684..7a9d681 100644 --- a/README.md +++ b/README.md @@ -131,18 +131,18 @@ By default the command starts the mount process in the background. To run it in DOCSPELL_CLI_MOUNT_FOREGROUND=1 docspell-cli mount ~/mnt/docspell ``` -Mounted files are grouped by Docspell folder and date: +Mounted files are organized by item date: ```text -///. +//. ``` -Items without a folder are placed below `No Folder`; items without a date are placed below `No Date`. Multiple attachments are named with a numeric suffix. File contents are downloaded lazily from Docspell when read. +The mount creates year and month directories without querying Docspell. Opening a month queries Docspell for that date, so documents added after mounting appear without a remount. Items without a date are placed below `No Date`. Multiple attachments and conflicting item names are given numeric suffixes. File contents are downloaded lazily from Docspell when read. The mount also includes: -- `by-id//.` entries for stable ID-based lookups. -- `.docspell-cli-metadata.json`, used by helper commands to map item IDs to friendly mounted paths. +- `by-id//.` entries for stable ID-based lookups. Every two-character Docspell ID prefix is shown; opening one queries only IDs with that prefix. +- `.docspell-cli-metadata.json`, which is empty for the lazy mount. Mount options: diff --git a/mount.go b/mount.go index 52d5b37..4a7abc9 100644 --- a/mount.go +++ b/mount.go @@ -53,6 +53,8 @@ type mountedItemMetadata struct { const mountMetadataFileName = ".docspell-cli-metadata.json" +const docspellIDCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" + func runMountCommand(args []string, cfg config, password string) error { options, err := parseMountOptions(args, cfg) if err != nil { @@ -72,11 +74,7 @@ func runMountCommand(args []string, cfg config, password string) error { if err := client.Login(context.Background(), cfg.User, password); err != nil { return err } - items, err := client.SearchItems(context.Background(), options.query, options.limit) - if err != nil { - return fmt.Errorf("search items: %w", err) - } - root := buildMountTree(context.Background(), client, items) + root := buildLazyMountTree(client, options.query, options.limit) conn, err := fuse.Mount( options.mountPoint, @@ -89,7 +87,7 @@ func runMountCommand(args []string, cfg config, password string) error { } defer conn.Close() - fmt.Printf("Mounted %d items read-only at %s. Unmount with: fusermount -u %s\n", len(items), options.mountPoint, options.mountPoint) + fmt.Printf("Mounted read-only at %s. Unmount with: fusermount -u %s\n", options.mountPoint, options.mountPoint) return fs.Serve(conn, &docspellFS{root: root}) } @@ -337,6 +335,255 @@ func itemIDFromLocalPath(path string, cfg config) (string, error) { return "", fmt.Errorf("could not map mounted path to item id") } +func buildLazyMountTree(client *DocspellClient, query string, limit int) *dirNode { + root := &dirNode{name: "", children: map[string]fs.Node{}} + currentYear := time.Now().Year() + for year := 1900; year <= currentYear+1; year++ { + name := fmt.Sprintf("%04d", year) + root.children[name] = &lazyYearNode{year: year, client: client, query: query, limit: limit} + } + root.children["No Date"] = &lazyItemDirNode{query: combineItemQueries(query, "not exist:date"), client: client, limit: limit} + root.children["by-id"] = &lazyByIDNode{client: client, query: query, limit: limit} + root.children[mountMetadataFileName] = &staticFileNode{ + name: mountMetadataFileName, + data: []byte("{\"items\":{}}\n"), + modTime: time.Now(), + } + return root +} + +type lazyYearNode struct { + year int + client *DocspellClient + query string + limit int +} + +type lazyMonthNode struct { + query string + client *DocspellClient + limit int +} + +type lazyItemDirNode struct { + query string + client *DocspellClient + limit int +} + +type lazyByIDNode struct { + client *DocspellClient + query string + limit int +} + +type lazyByIDShardNode struct { + query string + client *DocspellClient + limit int +} + +func combineItemQueries(base, constraint string) string { + base = strings.TrimSpace(base) + if base == "" { + return constraint + } + return "(& " + base + " " + constraint + " )" +} + +func (d *lazyYearNode) Attr(ctx context.Context, a *fuse.Attr) error { + a.Mode = os.ModeDir | 0555 + return nil +} + +func (d *lazyYearNode) Lookup(ctx context.Context, name string) (fs.Node, error) { + if len(name) != 2 || name < "01" || name > "12" { + return nil, fuse.ENOENT + } + return &lazyMonthNode{ + query: combineItemQueries(d.query, fmt.Sprintf("date:%04d-%s", d.year, name)), + client: d.client, + limit: d.limit, + }, nil +} + +func (d *lazyYearNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + entries := make([]fuse.Dirent, 0, 12) + for month := 1; month <= 12; month++ { + entries = append(entries, fuse.Dirent{Name: fmt.Sprintf("%02d", month), Type: fuse.DT_Dir}) + } + return entries, nil +} + +func (d *lazyMonthNode) Attr(ctx context.Context, a *fuse.Attr) error { + a.Mode = os.ModeDir | 0555 + return nil +} + +func (d *lazyMonthNode) Lookup(ctx context.Context, name string) (fs.Node, error) { + return (&lazyItemDirNode{query: d.query, client: d.client, limit: d.limit}).Lookup(ctx, name) +} + +func (d *lazyMonthNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + return (&lazyItemDirNode{query: d.query, client: d.client, limit: d.limit}).ReadDirAll(ctx) +} + +func (d *lazyItemDirNode) Attr(ctx context.Context, a *fuse.Attr) error { + a.Mode = os.ModeDir | 0555 + return nil +} + +func (d *lazyItemDirNode) children(ctx context.Context) (map[string]fs.Node, error) { + items, err := d.client.SearchItems(ctx, d.query, d.limit) + if err != nil { + return nil, fmt.Errorf("search items: %w", err) + } + root := buildMountTree(ctx, d.client, items) + children := map[string]fs.Node{} + for name, child := range root.children { + if name != "by-id" { + collectMountFiles(children, child) + } + } + return children, nil +} + +func collectMountFiles(children map[string]fs.Node, node fs.Node) { + dir, ok := node.(*dirNode) + if !ok { + file, ok := node.(*fileNode) + if ok { + name := file.name + for i := 2; ; i++ { + if _, exists := children[name]; !exists { + children[name] = file + break + } + ext := filepath.Ext(file.name) + name = fmt.Sprintf("%s (%d)%s", strings.TrimSuffix(file.name, ext), i, ext) + } + } + return + } + for _, child := range dir.children { + collectMountFiles(children, child) + } +} + +func (d *lazyItemDirNode) Lookup(ctx context.Context, name string) (fs.Node, error) { + children, err := d.children(ctx) + if err != nil { + return nil, err + } + child, ok := children[name] + if !ok { + return nil, fuse.ENOENT + } + return child, nil +} + +func (d *lazyItemDirNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + children, err := d.children(ctx) + if err != nil { + return nil, err + } + return dirEntries(children), nil +} + +func (d *lazyByIDNode) Attr(ctx context.Context, a *fuse.Attr) error { + a.Mode = os.ModeDir | 0555 + return nil +} + +func (d *lazyByIDNode) Lookup(ctx context.Context, name string) (fs.Node, error) { + if len(name) != 2 { + return nil, fuse.ENOENT + } + return &lazyByIDShardNode{ + query: combineItemQueries(d.query, "id:"+name+"*"), + client: d.client, + limit: d.limit, + }, nil +} + +func (d *lazyByIDNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + entries := make([]fuse.Dirent, 0, len(docspellIDCharacters)*len(docspellIDCharacters)) + for _, first := range docspellIDCharacters { + for _, second := range docspellIDCharacters { + entries = append(entries, fuse.Dirent{Name: string([]rune{first, second}), Type: fuse.DT_Dir}) + } + } + return entries, nil +} + +func (d *lazyByIDShardNode) Attr(ctx context.Context, a *fuse.Attr) error { + a.Mode = os.ModeDir | 0555 + return nil +} + +func (d *lazyByIDShardNode) children(ctx context.Context) (map[string]fs.Node, error) { + items, err := d.client.SearchItems(ctx, d.query, d.limit) + if err != nil { + return nil, fmt.Errorf("search items: %w", err) + } + root := buildMountTree(ctx, d.client, items) + byID, ok := root.children["by-id"].(*dirNode) + if !ok { + return map[string]fs.Node{}, nil + } + for _, child := range byID.children { + if shard, ok := child.(*dirNode); ok { + return shard.children, nil + } + } + return map[string]fs.Node{}, nil +} + +func (d *lazyByIDShardNode) Lookup(ctx context.Context, name string) (fs.Node, error) { + children, err := d.children(ctx) + if err != nil { + return nil, err + } + child, ok := children[name] + if !ok { + return nil, fuse.ENOENT + } + return child, nil +} + +func (d *lazyByIDShardNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + children, err := d.children(ctx) + if err != nil { + return nil, err + } + return dirEntries(children), nil +} + +func dirEntries(children map[string]fs.Node) []fuse.Dirent { + names := make([]string, 0, len(children)) + for name := range children { + names = append(names, name) + } + sort.Strings(names) + entries := make([]fuse.Dirent, 0, len(names)) + for _, name := range names { + typ := fuse.DT_File + if isDirectoryNode(children[name]) { + typ = fuse.DT_Dir + } + entries = append(entries, fuse.Dirent{Name: name, Type: typ}) + } + return entries +} + +func isDirectoryNode(node fs.Node) bool { + switch node.(type) { + case *dirNode, *lazyYearNode, *lazyMonthNode, *lazyItemDirNode, *lazyByIDNode, *lazyByIDShardNode: + return true + } + return false +} + // buildMountTree converts the pure item tree produced by buildItemTree into the // FUSE node tree served by the mount command. A single *fileNode is created per // *itemFile (shared between the friendly hierarchy and by-id lookups) so that a @@ -450,7 +697,7 @@ func (d *dirNode) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { entries := make([]fuse.Dirent, 0, len(names)) for _, name := range names { typ := fuse.DT_File - if _, ok := d.children[name].(*dirNode); ok { + if isDirectoryNode(d.children[name]) { typ = fuse.DT_Dir } entries = append(entries, fuse.Dirent{Name: name, Type: typ}) diff --git a/mount_test.go b/mount_test.go index 28a52e1..82e8d54 100644 --- a/mount_test.go +++ b/mount_test.go @@ -3,6 +3,8 @@ package main import ( "context" "encoding/json" + "net/http" + "net/http/httptest" "os" "path/filepath" "testing" @@ -10,6 +12,126 @@ import ( "bazil.org/fuse" ) +func TestLazyMountFetchesMonthOnlyWhenRead(t *testing.T) { + var queries []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request struct { + Query string `json:"query"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + queries = append(queries, request.Query) + items := []SearchItem{} + if len(queries) == 1 { + items = []SearchItem{{ + ID: "item-1", + Name: "Invoice", + Date: 1720396800000, + Attachments: []ItemAttachment{{ID: "att-1", Name: "invoice.pdf", Size: 1}}, + }} + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "groups": []map[string]any{{"items": items}}, + }) + })) + defer server.Close() + + root := buildLazyMountTree(NewDocspellClient(server.URL, ""), "state:confirmed", 100) + year, err := root.Lookup(context.Background(), "2024") + if err != nil { + t.Fatal(err) + } + month, err := year.(*lazyYearNode).Lookup(context.Background(), "07") + if err != nil { + t.Fatal(err) + } + if len(queries) != 0 { + t.Fatalf("expected no search before reading month, got %v", queries) + } + entries, err := month.(*lazyMonthNode).ReadDirAll(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name != "Invoice.pdf" { + t.Fatalf("unexpected month entries: %#v", entries) + } + if len(queries) != 2 || queries[0] != "(& state:confirmed date:2024-07 )" || queries[1] != "(& state:confirmed date:2024-07 )" { + t.Fatalf("unexpected queries: %v", queries) + } +} + +func TestLazyByIDShardFetchesMatchingPrefix(t *testing.T) { + var queries []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request struct { + Query string `json:"query"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + queries = append(queries, request.Query) + items := []SearchItem{} + if len(queries) == 1 { + items = []SearchItem{{ + ID: "it-item-1", + Name: "Invoice", + Attachments: []ItemAttachment{{ID: "att-1", Name: "invoice.pdf", Size: 1}}, + }} + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "groups": []map[string]any{{"items": items}}, + }) + })) + defer server.Close() + + root := buildLazyMountTree(NewDocspellClient(server.URL, ""), "", 100) + byID, err := root.Lookup(context.Background(), "by-id") + if err != nil { + t.Fatal(err) + } + shard, err := byID.(*lazyByIDNode).Lookup(context.Background(), "it") + if err != nil { + t.Fatal(err) + } + entries, err := shard.(*lazyByIDShardNode).ReadDirAll(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name != "it-item-1.pdf" { + t.Fatalf("unexpected shard entries: %#v", entries) + } + if len(queries) != 2 || queries[0] != "id:it*" || queries[1] != "id:it*" { + t.Fatalf("unexpected queries: %v", queries) + } +} + +func TestLazyByIDListsEveryPrefixWithoutSearching(t *testing.T) { + root := buildLazyMountTree(nil, "", 100) + byID, err := root.Lookup(context.Background(), "by-id") + if err != nil { + t.Fatal(err) + } + entries, err := byID.(*lazyByIDNode).ReadDirAll(context.Background()) + if err != nil { + t.Fatal(err) + } + wantCount := len(docspellIDCharacters) * len(docspellIDCharacters) + if len(entries) != wantCount { + t.Fatalf("expected %d ID prefixes, got %d", wantCount, len(entries)) + } + found := false + for _, entry := range entries { + if entry.Name == "7j" { + found = entry.Type == fuse.DT_Dir + break + } + } + if !found { + t.Fatalf("expected 7j directory in %#v", entries) + } +} + func TestBuildMountTreeGroupsByFolderAndDate(t *testing.T) { items := []SearchItem{{ ID: "item-1",