Merge pull request #4338 from broadinstitute/stricter-af-prefilter

Stricter AF prefilter

hail_search/fixtures/GRCh38/SNV_INDEL/high_af_variants.ht/README.txt

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
   Written with version 0.2.128-eead8100a1c1
-  Created at 2024/03/04 16:14:35
+  Created at 2024/08/29 13:43:52

hail_search/queries/snv_indel.py

@@ -24,7 +24,8 @@ class SnvIndelHailTableQuery(SnvIndelHailTableQuery37):
         SCREEN_KEY, MOTIF_FEATURES_KEY, REGULATORY_FEATURES_KEY,
     ]
     FREQUENCY_PREFILTER_FIELDS = OrderedDict([
-        (True, PREFILTER_FREQ_CUTOFF),
+        (True, 0.001),
+        ('is_gt_1_percent', PREFILTER_FREQ_CUTOFF),
         ('is_gt_3_percent', 0.03),
         ('is_gt_5_percent', 0.05),
         ('is_gt_10_percent', 0.1),

hail_search/queries/snv_indel_37.py

@@ -96,19 +96,26 @@ def _get_gnomad_af_prefilter(self, frequencies=None, pathogenicity=None, **kwarg
         if af_cutoff_field is None:
             return False
 
-        af_filter = True if af_cutoff_field is True else lambda ht: ht[af_cutoff_field]
-
+        clinvar_path_ht = False
         if af_cutoff < PATH_FREQ_OVERRIDE_CUTOFF:
             clinvar_path_ht = self._get_loaded_clinvar_prefilter_ht(pathogenicity)
-            if clinvar_path_ht is not False:
-                path_cutoff_field = self._get_af_prefilter_field(PATH_FREQ_OVERRIDE_CUTOFF)
-                non_clinvar_filter = lambda ht: hl.is_missing(clinvar_path_ht[ht.key])
-                if af_filter is not True:
-                    non_clinvar_filter = lambda ht: non_clinvar_filter(ht) & af_filter(ht)
-                af_filter = lambda ht: ht[path_cutoff_field] | non_clinvar_filter(ht)
+
+        if clinvar_path_ht is not False:
+            path_cutoff_field = self._get_af_prefilter_field(PATH_FREQ_OVERRIDE_CUTOFF)
+            non_clinvar_filter = lambda ht: hl.is_missing(clinvar_path_ht[ht.key])
+            if af_cutoff_field is not True:
+                non_clinvar_var_filter = non_clinvar_filter
+                non_clinvar_filter = lambda ht: non_clinvar_var_filter(ht) & self._af_prefilter(af_cutoff_field)(ht)
+            af_filter = lambda ht: ht[path_cutoff_field] | non_clinvar_filter(ht)
+        else:
+            af_filter = self._af_prefilter(af_cutoff_field)
 
         return af_filter
 
+    @staticmethod
+    def _af_prefilter(af_cutoff_field):
+        return True if af_cutoff_field is True else lambda ht: ht[af_cutoff_field]
+
     def _get_af_prefilter_field(self, af_cutoff):
         return next((field for field, cutoff in self.FREQUENCY_PREFILTER_FIELDS.items() if af_cutoff <= cutoff), None)