Fix Optimizer class when combining sub-filters

Refactored using simplified logic.
Added unit tests.

Fixes #13799.

Co-authored-by: Doug Blank <doug.blank@gmail.com>

gramps/gen/filters/__init__.py

@@ -42,5 +42,10 @@ def reload_custom_filters():
     CustomFilters.load()
 
 
+def set_custom_filters(filter_list):
+    global CustomFilters
+    CustomFilters = filter_list
+
+
 # if not CustomFilters:  # moved to viewmanager
 # reload_custom_filters()

gramps/gen/filters/_filterlist.py

@@ -25,6 +25,7 @@
 # Standard Python modules
 #
 # -------------------------------------------------------------------------
+from io import StringIO
 from xml.sax import make_parser, SAXParseException
 import os
 from collections import abc
@@ -115,6 +116,15 @@ class FilterList:
         except SAXParseException:
             print("Parser error")
 
+    def loadString(self, string):
+        """load a custom filter from the provided string"""
+        try:
+            parser = make_parser()
+            parser.setContentHandler(FilterParser(self))
+            parser.parse(StringIO(string))
+        except SAXParseException:
+            print("Parser error")
+
     def fix(self, line):
         """sanitize the custom filter name, if needed"""
         new_line = line.strip()

gramps/gen/filters/_genericfilter.py

@@ -4,6 +4,7 @@
 # Copyright (C) 2002-2006  Donald N. Allingham
 # Copyright (C) 2011       Tim G L Lyons
 # Copyright (C) 2012,2024  Doug Blank <doug.blank@gmail.com>
+# Copyright (C) 2025       Steve Youngs <steve@youngs.cc>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -24,6 +25,12 @@
 Package providing filtering framework for Gramps.
 """
 
+# -------------------------------------------------------------------------
+#
+# Standard Python modules
+#
+# -------------------------------------------------------------------------
+from __future__ import annotations
 import logging
 import time
 
@@ -46,6 +53,15 @@ from ..const import GRAMPS_LOCALE as glocale
 from .rules import Rule
 from .optimizer import Optimizer
 
+# -------------------------------------------------------------------------
+#
+# Typing modules
+#
+# -------------------------------------------------------------------------
+from typing import cast, Dict, List, Literal, Set, Tuple
+from ..db import Database
+from ..types import PrimaryObjectHandle
+
 _ = glocale.translation.gettext
 LOG = logging.getLogger(".filter.results")
 
@@ -56,9 +72,10 @@ LOG = logging.getLogger(".filter.results")
 #
 # -------------------------------------------------------------------------
 class GenericFilter:
-    """Filter class that consists of several rules."""
+    """Filter class that consists of zero, one or more rules."""
 
     logical_functions = ["and", "or", "one"]
+    logical_op: Literal["and", "or", "one"]
 
     def __init__(self, source=None):
         if source:
@@ -124,6 +141,9 @@ class GenericFilter:
     def get_rules(self):
         return self.flist
 
+    def get_all_handles(self, db):
+        return db.get_person_handles()
+
     def get_cursor(self, db):
         return db.get_person_cursor()
 
@@ -140,78 +160,44 @@ class GenericFilter:
         return db.get_number_of_people()
 
     def apply_logical_op_to_all(
-        self, db, id_list, apply_logical_op, user=None, tupleind=None, tree=False
+        self,
+        db,
+        possible_handles: Set[PrimaryObjectHandle],
+        apply_logical_op,
+        user=None,
     ):
-        final_list = []
-
-        optimizer = Optimizer(self)
-        handles_in, handles_out = optimizer.get_handles()
-
         LOG.debug(
-            "Optimizer handles_in: %s",
-            len(handles_in) if handles_in is not None else None,
+            "Starting possible_handles: %s",
+            len(possible_handles),
         )
-        LOG.debug("Optimizer handles_out: %s", len(handles_out))
-        if id_list is None:
-            if handles_in is not None:
-                if user:
-                    user.begin_progress(_("Filter"), _("Applying ..."), len(handles_in))
 
-                # Use these rather than going through entire database
-                for handle in handles_in:
-                    if user:
-                        user.step_progress()
+        # use the Optimizer to refine the set of possible_handles
+        optimizer = Optimizer()
+        handles_in, handles_out = optimizer.compute_potential_handles_for_filter(self)
 
-                    if handle is None:
-                        continue
+        # LOG.debug(
+        #    "Optimizer possible_handles: %s",
+        #    len(possible_handles),
+        # )
 
-                    obj = self.get_object(db, handle)
+        # if user:
+        #    user.begin_progress(_("Filter"), _("Applying ..."), len(possible_handles))
 
-                    if apply_logical_op(db, obj, self.flist) != self.invert:
-                        final_list.append(obj.handle)
+        # test each value in possible_handles to compute the final_list
+        final_list = []
+        for handle in possible_handles:
+            if handles_in is not None and handle not in handles_in:
+                continue
+            if handles_out is not None and handle in handles_out:
+                continue
 
-            else:
-                with (
-                    self.get_tree_cursor(db) if tree else self.get_cursor(db)
-                ) as cursor:
-                    if user:
-                        user.begin_progress(
-                            _("Filter"), _("Applying ..."), self.get_number(db)
-                        )
-
-                    for handle, obj in cursor:
-                        if user:
-                            user.step_progress()
-
-                        if handle in handles_out:
-                            continue
-
-                        if apply_logical_op(db, obj, self.flist) != self.invert:
-                            final_list.append(handle)
-
-        else:
-            id_list = list(id_list)
             if user:
-                user.begin_progress(_("Filter"), _("Applying ..."), len(id_list))
-            for handle_data in id_list:
-                if user:
-                    user.step_progress()
+                user.step_progress()
 
-                if tupleind is None:
-                    handle = handle_data
-                else:
-                    handle = handle_data[tupleind]
+            obj = self.get_object(db, handle)
 
-                if handles_in is not None:
-                    if handle not in handles_in:
-                        continue
-                elif handle in handles_out:
-                    continue
-
-                obj = self.get_object(db, handle)
-
-                if apply_logical_op(db, obj, self.flist) != self.invert:
-                    final_list.append(handle_data)
+            if apply_logical_op(db, obj, self.flist) != self.invert:
+                final_list.append(obj.handle)
 
         if user:
             user.end_progress()
@@ -250,15 +236,32 @@ class GenericFilter:
             raise Exception("invalid operator: %r" % self.logical_op)
         return res != self.invert
 
-    def apply(self, db, id_list=None, tupleind=None, user=None, tree=False):
+    def apply(
+        self,
+        db,
+        id_list: List[PrimaryObjectHandle | Tuple] | None = None,
+        tupleind: int | None = None,
+        user=None,
+        tree: bool = False,
+    ) -> List[PrimaryObjectHandle | Tuple]:
         """
         Apply the filter using db.
-        If id_list given, the handles in id_list are used. If not given
-        a database cursor will be used over all entries.
 
-        If tupleind is given, id_list is supposed to consist of a list of
-        tuples, with the handle being index tupleind. So
-        handle_0 = id_list[0][tupleind]
+        If id_list and tupleind given, id_list is a list of tuples. tupleind is the
+        index of the object handle. So handle_0 = id_list[0][tupleind]. The input
+        order in maintained in the output. If multiple tuples have the same handle,
+        it is undefined which tuple is returned.
+
+        If id_list given and tupleind is None, id_list if the list of handles to use
+
+        If id_list is None and tree is False, all handles in the database are searched.
+        The order of handles in the result is not guaranteed
+
+        If id_list is None and tree is True, all handles in the database are searched.
+        The order of handles in the result matches the order of traversal
+        by get_tree_cursor
+
+        id_list takes precendence over tree
 
         user is optional. If present it must be an instance of a User class.
 
@@ -282,9 +285,39 @@ class GenericFilter:
             raise Exception("invalid operator: %r" % self.logical_op)
 
         start_time = time.time()
-        res = self.apply_logical_op_to_all(
-            db, id_list, apply_logical_op, user, tupleind, tree
-        )
+
+        # build the starting set of possible_handles to be filtered
+        possible_handles: Set[PrimaryObjectHandle]
+        if id_list is not None:
+            if tupleind is not None:
+                # construct a dict from handle to corresponding tuple
+                # this is used to efficiently transform final_list from a list of
+                # handles to a list of tuples
+                handle_tuple: Dict[PrimaryObjectHandle, Tuple] = {
+                    data[tupleind]: data for data in cast(List[Tuple], id_list)
+                }
+                possible_handles = set(handle_tuple.keys())
+            else:
+                possible_handles = set(cast(List[PrimaryObjectHandle], id_list))
+        elif tree:
+            tree_handles = [handle for handle, obj in self.get_tree_cursor(db)]
+            possible_handles = set(tree_handles)
+        else:
+            possible_handles = set(self.get_all_handles(db))
+
+        res = self.apply_logical_op_to_all(db, possible_handles, apply_logical_op, user)
+
+        # convert the filtered set of handles to the correct result type
+        if id_list is not None and tupleind is not None:
+            # convert the final_list of handles back to the corresponding final_list of tuples
+            res = sorted(
+                [handle_tuple[handle] for handle in res],
+                key=lambda x: id_list.index(x),
+            )
+        elif tree:
+            # sort final_list into the same order as traversed by get_tree_cursor
+            res = sorted(res, key=lambda x: tree_handles.index(x))
+
         LOG.debug("Apply time: %s seconds", time.time() - start_time)
 
         for rule in self.flist:
@@ -300,6 +333,9 @@ class GenericFamilyFilter(GenericFilter):
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
+    def get_all_handles(self, db):
+        return db.get_family_handles()
+
     def get_cursor(self, db):
         return db.get_family_cursor()
 
@@ -320,6 +356,9 @@ class GenericEventFilter(GenericFilter):
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
+    def get_all_handles(self, db):
+        return db.get_event_handles()
+
     def get_cursor(self, db):
         return db.get_event_cursor()
 
@@ -340,6 +379,9 @@ class GenericSourceFilter(GenericFilter):
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
+    def get_all_handles(self, db):
+        return db.get_source_handles()
+
     def get_cursor(self, db):
         return db.get_source_cursor()
 
@@ -360,6 +402,9 @@ class GenericCitationFilter(GenericFilter):
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
+    def get_all_handles(self, db):
+        return db.get_citation_handles()
+
     def get_cursor(self, db):
         return db.get_citation_cursor()
 
@@ -383,6 +428,9 @@ class GenericPlaceFilter(GenericFilter):
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
+    def get_all_handles(self, db):
+        return db.get_place_handles()
+
     def get_cursor(self, db):
         return db.get_place_cursor()
 
@@ -406,6 +454,9 @@ class GenericMediaFilter(GenericFilter):
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
+    def get_all_handles(self, db):
+        return db.get_media_handles()
+
     def get_cursor(self, db):
         return db.get_media_cursor()
 
@@ -426,6 +477,9 @@ class GenericRepoFilter(GenericFilter):
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
+    def get_all_handles(self, db):
+        return db.get_repository_handles()
+
     def get_cursor(self, db):
         return db.get_repository_cursor()
 
@@ -446,6 +500,9 @@ class GenericNoteFilter(GenericFilter):
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
+    def get_all_handles(self, db):
+        return db.get_note_handles()
+
     def get_cursor(self, db):
         return db.get_note_cursor()
 

gramps/gen/filters/optimizer.py

@@ -2,6 +2,7 @@
 # Gramps - a GTK+/GNOME based genealogy program
 #
 # Copyright (C) 2024  Doug Blank <doug.blank@gmail.com>
+# Copyright (C) 2025  Steve Youngs <steve@youngs.cc>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,121 +18,66 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
-
+# -------------------------------------------------------------------------
+#
+# Standard Python modules
+#
+# -------------------------------------------------------------------------
+from __future__ import annotations
 import logging
 
+# -------------------------------------------------------------------------
+#
+# Typing modules
+#
+# -------------------------------------------------------------------------
+from typing import List, Set, TYPE_CHECKING, Tuple
+from .rules import Rule
+
+if TYPE_CHECKING:
+    from ._genericfilter import GenericFilter
+from ..types import PrimaryObjectHandle
+
 LOG = logging.getLogger(".filter.optimizer")
 
 
-def intersection(sets):
-    if sets:
-        result = sets[0]
-        for s in sets[1:]:
-            result = result.intersection(s)
-        return result
-    else:
-        return set()
-
-
-def union(sets):
-    if sets:
-        result = sets[0]
-        for s in sets[1:]:
-            result = result.union(s)
-        return result
-    else:
-        return set()
-
-
 class Optimizer:
-    """
-    Optimizer to use the filter's pre-selected selected_handles
-    to include or exclude.
-    """
+    def compute_potential_handles_for_filter(
+        self, filter: GenericFilter
+    ) -> Tuple[Set[PrimaryObjectHandle] | None, Set[PrimaryObjectHandle] | None]:
+        if len(filter.flist) == 0:
+            return (None, None)
 
-    def __init__(self, filter):
-        """
-        Initialize the collection of selected_handles in the filter list.
-        """
-        self.all_selected_handles = []
-        self.walk_filters(filter, False, self.all_selected_handles)
-
-    def walk_filters(self, filter, parent_invert, result):
-        """
-        Recursively walk all of the filters/rules and get
-        rules with selected_handles
-        """
-        current_invert = parent_invert if not filter.invert else not parent_invert
-        LOG.debug(
-            "walking, filter: %s, invert=%s, parent_invert=%s",
-            filter,
-            filter.invert,
-            parent_invert,
-        )
-        rules_with_selected_handles = []
-        for item in filter.flist:
-            if hasattr(item, "find_filter"):
-                rule_filter = item.find_filter()
-                if rule_filter is not None:
-                    self.walk_filters(
-                        rule_filter,
-                        current_invert,
-                        result,
-                    )
-            elif hasattr(item, "selected_handles"):
-                rules_with_selected_handles.append(set(item.selected_handles))
-        if rules_with_selected_handles:
-            LOG.debug(
-                "filter %s: parent_invert=%s, invert=%s, op=%s, number of rules with selected_handles=%s",
-                filter,
-                parent_invert,
-                filter.invert,
-                filter.logical_op,
-                len(rules_with_selected_handles),
-            )
-            result.append(
-                (
-                    current_invert,
-                    filter.logical_op,
-                    rules_with_selected_handles,
-                )
-            )
-
-    def get_handles(self):
-        """
-        Returns handles_in, and handles_out.
-
-        `handles_in` is either None, or a set of handles to include.
-        if it is None, then there is no evidence to only include
-        particular handles. If it is a set, then those in the set
-        are a superset of the items that will match.
-
-        `handles_out` is a set. If any handle is in the set, it will
-        not be included in the final results.
-
-        The handles_in are selected if all of the rules are connected with
-        "and" and not inverted. The handles_out are selected if all of the
-        rules are connected with "and" and inverted.
-        """
         handles_in = None
-        handles_out = set()
-        # Get all positive non-inverted selected_handles
-        for inverted, logical_op, selected_handles in self.all_selected_handles:
-            if logical_op == "and" and not inverted:
-                LOG.debug("optimizer positive match!")
-                if handles_in is None:
-                    handles_in = intersection(selected_handles)
-                else:
-                    handles_in = intersection([handles_in] + selected_handles)
+        handles_out = None
+        for rule in filter.flist:
+            if filter.logical_op == "and" or len(filter.flist) == 1:
+                rule_in, rule_out = self.compute_potential_handles_for_rule(rule)
+                if rule_in is not None:
+                    if handles_in is None:
+                        handles_in = rule_in
+                    else:
+                        handles_in = handles_in.intersection(rule_in)
+                if rule_out is not None:
+                    if handles_out is None:
+                        handles_out = rule_out
+                    else:
+                        handles_out = handles_out.union(rule_out)
 
-        # Get all inverted selected_handles:
-        for inverted, logical_op, selected_handles in self.all_selected_handles:
-            if logical_op == "and" and inverted:
-                LOG.debug("optimizer inverted match!")
-                handles_out = union([handles_out] + selected_handles)
+        if filter.invert:
+            handles_in, handles_out = handles_out, handles_in
 
-        if handles_in is not None:
-            handles_in = handles_in - handles_out
-
-        LOG.debug("optimizer handles_in: %s", len(handles_in) if handles_in else 0)
         return handles_in, handles_out
+
+    def compute_potential_handles_for_rule(
+        self,
+        rule: Rule,
+    ) -> Tuple[Set[PrimaryObjectHandle] | None, Set[PrimaryObjectHandle] | None]:
+        """ """
+        if hasattr(rule, "selected_handles"):
+            return (rule.selected_handles, None)
+        if hasattr(rule, "find_filter"):
+            filter = rule.find_filter()
+            if filter:
+                return self.compute_potential_handles_for_filter(filter)
+        return (None, None)

gramps/gen/filters/rules/person/_isdescendantfamilyof.py

@@ -114,7 +114,8 @@ class IsDescendantFamilyOf(Rule):
                         spouse_handle = family.mother_handle
                     else:
                         spouse_handle = family.father_handle
-                    self.selected_handles.add(spouse_handle)
+                    if spouse_handle:
+                        self.selected_handles.add(spouse_handle)
 
     def exclude(self):
         # This removes root person and his/her spouses from the matches set

gramps/gen/filters/rules/test/place_rules_test.py

@@ -70,13 +70,13 @@ class BaseTest(unittest.TestCase):
         """
         cls.db = import_as_dict(EXAMPLE, User())
 
-    def filter_with_rule(self, rule):
+    def filter_with_rule(self, rule, tree=False):
         """
         Apply a filter with the given rule.
         """
         filter_ = GenericPlaceFilter()
         filter_.add_rule(rule)
-        results = filter_.apply(self.db)
+        results = filter_.apply(self.db, tree=tree)
         return set(results)
 
     def test_allplaces(self):
@@ -88,6 +88,15 @@ class BaseTest(unittest.TestCase):
             len(self.filter_with_rule(rule)), self.db.get_number_of_places()
         )
 
+    def test_hascitation_with_tree(self):
+        """
+        Test AllPlaces rule.
+        """
+        rule = HasCitation(["page 23", "", ""])
+        self.assertEqual(
+            self.filter_with_rule(rule, tree=True), set(["YNUJQC8YM5EGRG868J"])
+        )
+
     def test_hascitation(self):
         """
         Test HasCitation rule.

gramps/gen/filters/test/filter_optimizer_test.py

@@ -0,0 +1,427 @@
+#
+# Gramps - a GTK+/GNOME based genealogy program
+#
+# Copyright (C) 2025 Doug Blank <doug.blank@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+import tempfile
+import os
+import unittest
+
+from ...const import DATA_DIR
+from ...db.utils import import_as_dict
+from ...user import User
+from ...filters import reload_custom_filters, FilterList, set_custom_filters
+from ....gen import filters
+
+custom_filters_xml = """<?xml version="1.0" encoding="utf-8"?>
+<filters>
+  <object type="Person">
+    <filter name="Ancestors of" function="and">
+      <rule class="IsAncestorOf" use_regex="False" use_case="False">
+        <arg value="I0044"/>
+        <arg value="1"/>
+      </rule>
+    </filter>
+    <filter name="Family and their Spouses" function="or">
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="Ancestors of"/>
+      </rule>
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="Siblings of Ancestors"/>
+      </rule>
+    </filter>
+    <filter name="Siblings of Ancestors" function="and">
+      <rule class="IsSiblingOfFilterMatch" use_regex="False" use_case="False">
+        <arg value="Ancestors of"/>
+      </rule>
+    </filter>
+    <filter name="Everyone" function="or">
+      <rule class="Everyone" use_regex="False" use_case="False">
+      </rule>
+    </filter>
+    <filter name="F1" function="or">
+      <rule class="Everyone" use_regex="False" use_case="False">
+      </rule>
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="F2"/>
+      </rule>
+    </filter>
+    <filter name="F2" function="and">
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="F3"/>
+      </rule>
+    </filter>
+    <filter name="F3" function="and">
+      <rule class="IsAncestorOf" use_regex="False" use_case="False">
+        <arg value="I0041"/>
+        <arg value="0"/>
+      </rule>
+    </filter>
+    <filter name="I0001 xor I0002" function="one">
+      <rule class="HasIdOf" use_regex="False" use_case="False">
+        <arg value="I0001"/>
+      </rule>
+      <rule class="HasIdOf" use_regex="False" use_case="False">
+        <arg value="I0002"/>
+      </rule>
+    </filter>
+    <filter name="I0001 or I0002" function="or">
+      <rule class="HasIdOf" use_regex="False" use_case="False">
+        <arg value="I0001"/>
+      </rule>
+      <rule class="HasIdOf" use_regex="False" use_case="False">
+        <arg value="I0002"/>
+      </rule>
+    </filter>
+    <filter name="I0001 and I0002" function="and">
+      <rule class="HasIdOf" use_regex="False" use_case="False">
+        <arg value="I0001"/>
+      </rule>
+      <rule class="HasIdOf" use_regex="False" use_case="False">
+        <arg value="I0002"/>
+      </rule>
+    </filter>
+    <filter name="I0001" function="or">
+      <rule class="HasIdOf" use_regex="False" use_case="False">
+        <arg value="I0001"/>
+      </rule>
+    </filter>
+    <filter name="I0002" function="or">
+      <rule class="HasIdOf" use_regex="False" use_case="False">
+        <arg value="I0002"/>
+      </rule>
+    </filter>
+    <filter name="not I0001" function="or" invert="1">
+      <rule class="HasIdOf" use_regex="False" use_case="False">
+        <arg value="I0001"/>
+      </rule>
+    </filter>
+    <filter name="not I0002" function="or" invert="1">
+      <rule class="HasIdOf" use_regex="False" use_case="False">
+        <arg value="I0002"/>
+      </rule>
+    </filter>
+    <filter name="(not I0001) and (not I0002)" function="and">
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="not I0001"/>
+      </rule>
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="not I0002"/>
+      </rule>
+    </filter>
+    <filter name="(not I0001) or (not I0002)" function="or">
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="not I0001"/>
+      </rule>
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="not I0002"/>
+      </rule>
+    </filter>
+
+    <filter name="not ((not I0001) and (not I0002))" function="and" invert="1">
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="not I0001"/>
+      </rule>
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="not I0002"/>
+      </rule>
+    </filter>
+
+    <filter name="not (I0001 and I0002)" function="and" invert="1">
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="I0001"/>
+      </rule>
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="I0002"/>
+      </rule>
+    </filter>
+    <filter name="Empty Filter and" function="and">
+    </filter>
+    <filter name="Empty Filter and invert" function="and" invert="1">
+    </filter>
+    <filter name="Empty Filter or" function="or">
+    </filter>
+    <filter name="Empty Filter or invert" function="or" invert="1">
+    </filter>
+    <filter name="Empty Filter one" function="one">
+    </filter>
+    <filter name="Empty Filter one invert" function="one" invert="1">
+    </filter>
+    <filter name="Tag = ToDo" function="and">
+      <rule class="HasTag" use_regex="False" use_case="False">
+        <arg value="ToDo"/>
+      </rule>
+    </filter>
+    <filter name="Tag = ToDo Invert" function="and" invert="1">
+      <rule class="HasTag" use_regex="False" use_case="False">
+        <arg value="ToDo"/>
+      </rule>
+    </filter>
+    <filter name="Family Name" function="and">
+      <rule class="HasNameOf" use_regex="False" use_case="False">
+        <arg value=""/>
+        <arg value=""/>
+        <arg value=""/>
+        <arg value=""/>
+        <arg value=""/>
+        <arg value=""/>
+        <arg value=""/>
+        <arg value="Garner"/>
+        <arg value=""/>
+        <arg value=""/>
+        <arg value=""/>
+      </rule>
+    </filter>
+    <filter name="NOT Family Name" function="and" invert="1">
+      <rule class="MatchesFilter" use_regex="False" use_case="False">
+        <arg value="Family Name"/>
+      </rule>
+    </filter>
+    <filter name="Person I1041" function="and">
+      <rule class="HasIdOf" use_regex="False" use_case="False">
+        <arg value="I1041"/>
+      </rule>
+    </filter>
+    <filter name="Parents of Person I1041" function="or">
+      <rule class="IsParentOfFilterMatch" use_regex="False" use_case="False">
+        <arg value="Person I1041"/>
+      </rule>
+    </filter>
+    <filter name="not Parents of Person I1041" function="or" invert="1">
+      <rule class="IsParentOfFilterMatch" use_regex="False" use_case="False">
+        <arg value="Person I1041"/>
+      </rule>
+    </filter>
+  </object>
+</filters>
+"""
+TEST_DIR = os.path.abspath(os.path.join(DATA_DIR, "tests"))
+EXAMPLE = os.path.join(TEST_DIR, "example.gramps")
+
+
+class OptimizerTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        """
+        Import example database.
+        """
+        cls.db = import_as_dict(EXAMPLE, User())
+
+        fl = FilterList("")
+        fl.loadString(custom_filters_xml)
+
+        filters.set_custom_filters(fl)
+        cls.the_custom_filters = filters.CustomFilters.get_filters_dict("Person")
+        cls.filters = fl.get_filters_dict("Person")
+        for filter_name in cls.filters:
+            cls.the_custom_filters[filter_name] = cls.filters[filter_name]
+
+    @classmethod
+    def tearDownClass(self):
+        reload_custom_filters()
+
+    def test_ancestors_of(self):
+        filter = self.filters["Ancestors of"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 7)
+
+    def test_siblings_of_ancestors(self):
+        filter = self.filters["Siblings of Ancestors"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 18)
+
+    def test_family_and_their_spouses(self):
+        filter = self.filters["Family and their Spouses"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 25)
+
+    def test_everyone(self):
+        filter = self.filters["Everyone"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2128)
+
+    def test_everyone_with_id_list(self):
+        filter = self.filters["Everyone"]
+        id_list = [self.db.get_default_handle()]
+        results = filter.apply(self.db, id_list=id_list)
+        self.assertEqual(len(results), 1)
+
+    def test_everyone_with_id_list_tupleind_0(self):
+        filter = self.filters["Everyone"]
+        default_person = self.db.get_default_person()
+        id_list = [(default_person.handle, default_person)]
+        results = filter.apply(self.db, id_list=id_list, tupleind=0)
+        self.assertEqual(results[0], id_list[0])
+        self.assertEqual(len(results), 1)
+
+    def test_everyone_with_id_list_tupleind_1(self):
+        filter = self.filters["Everyone"]
+        default_person = self.db.get_default_person()
+        id_list = [(default_person, default_person.handle)]
+        results = filter.apply(self.db, id_list=id_list, tupleind=1)
+        self.assertEqual(results[0], id_list[0])
+        self.assertEqual(len(results), 1)
+
+    def test_everyone_with_id_list_tupleind_3(self):
+        filter = self.filters["Everyone"]
+        default_person = self.db.get_default_person()
+        id_list = [(default_person, default_person.handle)]
+        self.assertRaises(
+            IndexError, filter.apply, self.db, id_list=id_list, tupleind=3
+        )
+
+    def test_everyone_with_id_list_empty(self):
+        filter = self.filters["Everyone"]
+        id_list = []
+        results = filter.apply(self.db, id_list=id_list)
+        self.assertEqual(len(results), 0)
+
+    def test_f1(self):
+        filter = self.filters["F1"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2128)
+
+    def test_f2(self):
+        filter = self.filters["F2"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 16)
+
+    def test_f3(self):
+        filter = self.filters["F3"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 16)
+
+    def test_I0001xorI0002(self):
+        filter = self.filters["I0001 xor I0002"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2)
+
+    def test_I0001orI0002(self):
+        filter = self.filters["I0001 or I0002"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2)
+
+    def test_I0001andI0002(self):
+        filter = self.filters["I0001 and I0002"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 0)
+
+    def test_EmptyFilter_and(self):
+        filter = self.filters["Empty Filter and"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2128)
+
+    def test_EmptyFilter_and_invert(self):
+        filter = self.filters["Empty Filter and invert"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 0)
+
+    def test_EmptyFilter_or(self):
+        filter = self.filters["Empty Filter or"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 0)
+
+    def test_EmptyFilter_or_invert(self):
+        filter = self.filters["Empty Filter or invert"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2128)
+
+    def test_EmptyFilter_one(self):
+        filter = self.filters["Empty Filter one"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 0)
+
+    def test_EmptyFilter_one_invert(self):
+        filter = self.filters["Empty Filter one invert"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2128)
+
+    def test_TagToDo(self):
+        filter = self.filters["Tag = ToDo"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 1)
+
+    def test_TagToDo_invert(self):
+        filter = self.filters["Tag = ToDo Invert"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2127)
+
+    def test_family_name(self):
+        filter = self.filters["Family Name"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 71)
+
+    def test_not_family_name(self):
+        filter = self.filters["NOT Family Name"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2128 - 71)
+
+    def test_not_i0001_and_not_i0002(self):
+        filter = self.filters["(not I0001) and (not I0002)"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2128 - 2)
+
+    def test_not_i0001_or_not_i0002(self):
+        filter = self.filters["(not I0001) or (not I0002)"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2128)
+
+    def test_not_not_i0001_and_not_i0002(self):
+        filter = self.filters["not ((not I0001) and (not I0002))"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2)
+
+    def test_not_i0001_and_i0002(self):
+        filter = self.filters["not (I0001 and I0002)"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2128)
+
+    def test_i0001(self):
+        filter = self.filters["I0001"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 1)
+
+    def test_not_i0001(self):
+        filter = self.filters["not I0001"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2127)
+
+    def test_i0002(self):
+        filter = self.filters["I0002"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 1)
+
+    def test_not_i0002(self):
+        filter = self.filters["not I0002"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2127)
+
+    def test_person_I1041(self):
+        filter = self.filters["Person I1041"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 1)
+
+    def test_parents_of_person_I1041(self):
+        filter = self.filters["Parents of Person I1041"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2)
+
+    def test_not_parents_of_person_I1041(self):
+        filter = self.filters["not Parents of Person I1041"]
+        results = filter.apply(self.db)
+        self.assertEqual(len(results), 2128 - 2)

gramps/plugins/test/reports_test.py

@@ -90,7 +90,7 @@ def dynamic_report_method(report_name, test_function, files, *args, **options):
 
     def test_method(self):  # This needs to have "test" in name
         out, err = self.call(*args)
-        self.assertTrue(test_function(out, err, report_name, **options))
+        self.assertTrue(test_function(out, err, report_name, **options), out + err)
 
     return test_method
 
@@ -100,7 +100,7 @@ def dynamic_cli_method(report_name, test_function, files, *args, **options):
 
     def test_method(self):  # This needs to have "test" in name
         out, err = self.call(*args)
-        self.assertTrue(test_function(out, err, report_name, **options))
+        self.assertTrue(test_function(out, err, report_name, **options), out + err)
 
     return test_method
 

po/POTFILES.skip

@@ -154,6 +154,11 @@ gramps/gen/filters/rules/test/person_rules_test.py
 gramps/gen/filters/rules/test/place_rules_test.py
 gramps/gen/filters/rules/test/repository_rules_test.py
 #
+# gen.filters.test package
+#
+gramps/gen/filters/test/__init__.py
+gramps/gen/filters/test/filter_optimizer_test.py
+#
 # gen lib API
 #
 gramps/gen/lib/__init__.py