threading: enable filtering of server-side threads

This patch enables the filtering of a threaded view which uses
server-built threads. Filtering is done server-side, in order to
preserve the use of server-built threads.

In adding this feature, the filtering of notmuch folders was brought up
to feature parity with the other workers. The filters function the same
(ie: they can be stacked). The notmuch filters, however, still use
notmuch syntax for the filtering.

Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
Acked-by: Robin Jarry <robin@jarry.cc>

10 files changed, 103 insertions, 77 deletions

diff --git a/commands/account/search.go b/commands/account/search.go
index 771c850..7cae227 100644
--- a/commands/account/search.go
+++ b/commands/account/search.go
@@ -6,6 +6,7 @@ import (
 
 	"git.sr.ht/~rjarry/aerc/lib/statusline"
 	"git.sr.ht/~rjarry/aerc/widgets"
+	"git.sr.ht/~rjarry/aerc/worker/types"
 )
 
 type SearchFilter struct{}
@@ -32,27 +33,29 @@ func (SearchFilter) Execute(aerc *widgets.Aerc, args []string) error {
 		return errors.New("Cannot perform action. Messages still loading")
 	}
 
-	var cb func([]uint32)
 	if args[0] == "filter" {
 		if len(args[1:]) == 0 {
 			return Clear{}.Execute(aerc, []string{"clear"})
 		}
 		acct.SetStatus(statusline.FilterActivity("Filtering..."), statusline.Search(""))
-		cb = func(uids []uint32) {
-			acct.SetStatus(statusline.FilterResult(strings.Join(args, " ")))
-			acct.Logger().Printf("Filter results: %v", uids)
-			store.ApplyFilter(uids)
+		store.SetFilter(args[1:])
+		cb := func(msg types.WorkerMessage) {
+			if _, ok := msg.(*types.Done); ok {
+				acct.SetStatus(statusline.FilterResult(strings.Join(args, " ")))
+				acct.Logger().Printf("Filter results: %v", store.Uids())
+			}
 		}
+		store.Sort(nil, cb)
 	} else {
 		acct.SetStatus(statusline.Search("Searching..."))
-		cb = func(uids []uint32) {
+		cb := func(uids []uint32) {
 			acct.SetStatus(statusline.Search(strings.Join(args, " ")))
 			acct.Logger().Printf("Search results: %v", uids)
 			store.ApplySearch(uids)
 			// TODO: Remove when stores have multiple OnUpdate handlers
 			acct.Messages().Invalidate()
 		}
+		store.Search(args, cb)
 	}
-	store.Search(args, cb)
 	return nil
 }
diff --git a/commands/account/sort.go b/commands/account/sort.go
index a35ff33..e9ee4a3 100644
--- a/commands/account/sort.go
+++ b/commands/account/sort.go
@@ -84,8 +84,10 @@ func (Sort) Execute(aerc *widgets.Aerc, args []string) error {
 	}
 
 	acct.SetStatus(statusline.Sorting(true))
-	store.Sort(sortCriteria, func() {
-		acct.SetStatus(statusline.Sorting(false))
+	store.Sort(sortCriteria, func(msg types.WorkerMessage) {
+		if _, ok := msg.(*types.Done); ok {
+			acct.SetStatus(statusline.Sorting(false))
+		}
 	})
 	return nil
 }
diff --git a/doc/aerc-notmuch.5.scd b/doc/aerc-notmuch.5.scd
index 44b6b9d..7da7ecf 100644
--- a/doc/aerc-notmuch.5.scd
+++ b/doc/aerc-notmuch.5.scd
@@ -74,10 +74,14 @@ in notmuch, like :delete and :archive.++
 Others are slightly different in semantics and mentioned below:
 
 *cf* <notmuch query>
-	The change folder command allows for arbitrary notmuch queries and should
-	usually be preferred over *:filter* as it will be much faster if you use
-	the notmuch database to do the filtering
-
+	The change folder command allows for arbitrary notmuch queries. Performing a
+	cf command will perform a new top-level notmuch query
+
+*filter* <notmuch query>
+	The filter command for notmuch backends takes in arbitrary notmuch queries.
+	It applies the query on the set of messages shown in the message list. This
+	can be used to perform successive filters/queries. It is equivalent to
+	performing a set of queries concatenated with "and"
 
 # SEE ALSO
 
diff --git a/lib/msgstore.go b/lib/msgstore.go
index 99ae594..2e77369 100644
--- a/lib/msgstore.go
+++ b/lib/msgstore.go
@@ -33,8 +33,7 @@ type MessageStore struct {
 	// Search/filter results
 	results     []uint32
 	resultIndex int
-	filtered    []uint32
-	filter      bool
+	filter      []string
 
 	sortCriteria []*types.SortCriterion
 
@@ -83,6 +82,7 @@ func NewMessageStore(worker *types.Worker,
 		threadedView: thread,
 		buildThreads: clientThreads,
 
+		filter:       []string{"filter"},
 		sortCriteria: defaultSortCriteria,
 
 		pendingBodies:  make(map[uint32]interface{}),
@@ -215,7 +215,6 @@ func (store *MessageStore) Update(msg types.WorkerMessage) {
 		}
 		store.Messages = newMap
 		store.uids = msg.Uids
-		sort.SortBy(store.filtered, store.uids)
 		store.checkMark()
 		update = true
 	case *types.DirectoryThreaded:
@@ -313,14 +312,6 @@ func (store *MessageStore) Update(msg types.WorkerMessage) {
 		}
 		store.results = newResults
 
-		var newFiltered []uint32
-		for _, res := range store.filtered {
-			if _, deleted := toDelete[res]; !deleted {
-				newFiltered = append(newFiltered, res)
-			}
-		}
-		store.filtered = newFiltered
-
 		for _, thread := range store.Threads {
 			thread.Walk(func(t *types.Thread, _ int, _ error) error {
 				if _, deleted := toDelete[t.Uid]; deleted {
@@ -392,13 +383,7 @@ func (store *MessageStore) runThreadBuilder() {
 			store.builder.Update(msg)
 		}
 	}
-	var uids []uint32
-	if store.filter {
-		uids = store.filtered
-	} else {
-		uids = store.uids
-	}
-	store.Threads = store.builder.Threads(uids)
+	store.Threads = store.builder.Threads(store.uids)
 }
 
 func (store *MessageStore) Delete(uids []uint32,
@@ -496,10 +481,6 @@ func (store *MessageStore) Uids() []uint32 {
 			return uids
 		}
 	}
-
-	if store.filter {
-		return store.filtered
-	}
 	return store.uids
 }
 
@@ -732,30 +713,16 @@ func (store *MessageStore) ApplySearch(results []uint32) {
 	store.NextResult()
 }
 
-func (store *MessageStore) ApplyFilter(results []uint32) {
-	defer store.Reselect(store.Selected())
-	store.results = nil
-	store.filtered = results
-	store.filter = true
-	if store.onFilterChange != nil {
-		store.onFilterChange(store)
-	}
-	store.update()
-	// any marking is now invalid
-	// TODO: could save that probably
-	store.ClearVisualMark()
+func (store *MessageStore) SetFilter(args []string) {
+	store.filter = append(store.filter, args...)
 }
 
 func (store *MessageStore) ApplyClear() {
-	store.results = nil
-	store.filtered = nil
-	store.filter = false
-	if store.BuildThreads() {
-		store.runThreadBuilder()
-	}
+	store.filter = []string{"filter"}
 	if store.onFilterChange != nil {
 		store.onFilterChange(store)
 	}
+	store.Sort(nil, nil)
 }
 
 func (store *MessageStore) nextPrevResult(delta int) {
@@ -796,24 +763,30 @@ func (store *MessageStore) ModifyLabels(uids []uint32, add, remove []string,
 	}, cb)
 }
 
-func (store *MessageStore) Sort(criteria []*types.SortCriterion, cb func()) {
+func (store *MessageStore) Sort(criteria []*types.SortCriterion, cb func(types.WorkerMessage)) {
+	if criteria == nil {
+		criteria = store.sortCriteria
+	} else {
+		store.sortCriteria = criteria
+	}
 	store.Sorting = true
-	store.sortCriteria = criteria
 
 	handle_return := func(msg types.WorkerMessage) {
 		store.Sorting = false
 		if cb != nil {
-			cb()
+			cb(msg)
 		}
 	}
 
 	if store.threadedView && !store.buildThreads {
 		store.worker.PostAction(&types.FetchDirectoryThreaded{
-			SortCriteria: criteria,
+			SortCriteria:   criteria,
+			FilterCriteria: store.filter,
 		}, handle_return)
 	} else {
 		store.worker.PostAction(&types.FetchDirectoryContents{
-			SortCriteria: criteria,
+			SortCriteria:   criteria,
+			FilterCriteria: store.filter,
 		}, handle_return)
 	}
 }
diff --git a/widgets/dirlist.go b/widgets/dirlist.go
index ad5f048..5c3416c 100644
--- a/widgets/dirlist.go
+++ b/widgets/dirlist.go
@@ -191,7 +191,11 @@ func (dirlist *DirectoryList) Select(name string) {
 						}
 						dirlist.sortDirsByFoldersSortConfig()
 						if newStore {
-							dirlist.worker.PostAction(&types.FetchDirectoryContents{}, nil)
+							store, ok := dirlist.MsgStore(name)
+							if ok {
+								// Fetch directory contents via store.Sort
+								store.Sort(nil, nil)
+							}
 						}
 					}
 					dirlist.Invalidate()
diff --git a/worker/imap/open.go b/worker/imap/open.go
index a0607d0..b52a3c6 100644
--- a/worker/imap/open.go
+++ b/worker/imap/open.go
@@ -3,7 +3,6 @@ package imap
 import (
 	"sort"
 
-	"github.com/emersion/go-imap"
 	sortthread "github.com/emersion/go-imap-sortthread"
 
 	"git.sr.ht/~rjarry/aerc/worker/types"
@@ -29,11 +28,13 @@ func (imapw *IMAPWorker) handleFetchDirectoryContents(
 
 	imapw.worker.Logger.Printf("Fetching UID list")
 
-	seqSet := &imap.SeqSet{}
-	seqSet.AddRange(1, imapw.selected.Messages)
-
-	searchCriteria := &imap.SearchCriteria{
-		SeqNum: seqSet,
+	searchCriteria, err := parseSearch(msg.FilterCriteria)
+	if err != nil {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+		return
 	}
 	sortCriteria := translateSortCriterions(msg.SortCriteria)
 
@@ -98,10 +99,16 @@ func (imapw *IMAPWorker) handleDirectoryThreaded(
 	msg *types.FetchDirectoryThreaded) {
 	imapw.worker.Logger.Printf("Fetching threaded UID list")
 
-	seqSet := &imap.SeqSet{}
-	seqSet.AddRange(1, imapw.selected.Messages)
+	searchCriteria, err := parseSearch(msg.FilterCriteria)
+	if err != nil {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+		return
+	}
 	threads, err := imapw.client.thread.UidThread(sortthread.References,
-		&imap.SearchCriteria{SeqNum: seqSet})
+		searchCriteria)
 	if err != nil {
 		imapw.worker.PostMessage(&types.Error{
 			Message: types.RespondTo(msg),
diff --git a/worker/imap/search.go b/worker/imap/search.go
index f866b1c..46a25c7 100644
--- a/worker/imap/search.go
+++ b/worker/imap/search.go
@@ -11,6 +11,9 @@ import (
 
 func parseSearch(args []string) (*imap.SearchCriteria, error) {
 	criteria := imap.NewSearchCriteria()
+	if len(args) == 0 {
+		return criteria, nil
+	}
 
 	opts, optind, err := getopt.Getopts(args, "rubax:X:t:H:f:c:")
 	if err != nil {
diff --git a/worker/maildir/worker.go b/worker/maildir/worker.go
index 0862838..cc03ec8 100644
--- a/worker/maildir/worker.go
+++ b/worker/maildir/worker.go
@@ -406,10 +406,25 @@ func (w *Worker) handleOpenDirectory(msg *types.OpenDirectory) error {
 
 func (w *Worker) handleFetchDirectoryContents(
 	msg *types.FetchDirectoryContents) error {
-	uids, err := w.c.UIDs(*w.selected)
-	if err != nil {
-		w.worker.Logger.Printf("error scanning uids: %v", err)
-		return err
+	var (
+		uids []uint32
+		err  error
+	)
+	if len(msg.FilterCriteria) > 0 {
+		filter, err := parseSearch(msg.FilterCriteria)
+		if err != nil {
+			return err
+		}
+		uids, err = w.search(filter)
+		if err != nil {
+			return err
+		}
+	} else {
+		uids, err = w.c.UIDs(*w.selected)
+		if err != nil {
+			w.worker.Logger.Printf("error scanning uids: %v", err)
+			return err
+		}
 	}
 	sortedUids, err := w.sort(uids, msg.SortCriteria)
 	if err != nil {
diff --git a/worker/notmuch/worker.go b/worker/notmuch/worker.go
index 35e6840..27be73d 100644
--- a/worker/notmuch/worker.go
+++ b/worker/notmuch/worker.go
@@ -550,7 +550,14 @@ func (w *worker) loadExcludeTags(
 }
 
 func (w *worker) emitDirectoryContents(parent types.WorkerMessage) error {
-	uids, err := w.uidsFromQuery(w.query)
+	query := w.query
+	if msg, ok := parent.(*types.FetchDirectoryContents); ok {
+		s := strings.Join(msg.FilterCriteria[1:], " ")
+		if s != "" {
+			query = fmt.Sprintf("(%v) and (%v)", query, s)
+		}
+	}
+	uids, err := w.uidsFromQuery(query)
 	if err != nil {
 		return fmt.Errorf("could not fetch uids: %v", err)
 	}
@@ -567,12 +574,18 @@ func (w *worker) emitDirectoryContents(parent types.WorkerMessage) error {
 }
 
 func (w *worker) emitDirectoryThreaded(parent types.WorkerMessage) error {
-	threads, err := w.db.ThreadsFromQuery(w.query)
+	query := w.query
+	if msg, ok := parent.(*types.FetchDirectoryThreaded); ok {
+		s := strings.Join(msg.FilterCriteria[1:], " ")
+		if s != "" {
+			query = fmt.Sprintf("(%v) and (%v)", query, s)
+		}
+	}
+	threads, err := w.db.ThreadsFromQuery(query)
 	if err != nil {
 		return err
 	}
 	w.w.PostMessage(&types.DirectoryThreaded{
-		Message: types.RespondTo(parent),
 		Threads: threads,
 	}, nil)
 	return nil
diff --git a/worker/types/messages.go b/worker/types/messages.go
index e303ade..a414eb2 100644
--- a/worker/types/messages.go
+++ b/worker/types/messages.go
@@ -87,12 +87,14 @@ type OpenDirectory struct {
 
 type FetchDirectoryContents struct {
 	Message
-	SortCriteria []*SortCriterion
+	SortCriteria   []*SortCriterion
+	FilterCriteria []string
 }
 
 type FetchDirectoryThreaded struct {
 	Message
-	SortCriteria []*SortCriterion
+	SortCriteria   []*SortCriterion
+	FilterCriteria []string
 }
 
 type SearchDirectory struct {