Merge branch 'main' into export-shelx-complete-ins

.conda-envs/linux.txt

@@ -19,6 +19,7 @@ conda-forge::jpeg
 conda-forge::libboost-devel
 conda-forge::libboost-python-devel
 conda-forge::matplotlib-base>=3.0.2
+conda-forge::mesa-libgl-devel-cos7-x86_64
 conda-forge::mrcfile
 conda-forge::msgpack-cxx
 conda-forge::msgpack-python
@@ -50,6 +51,6 @@ conda-forge::sqlite
 conda-forge::tabulate
 conda-forge::tqdm
 conda-forge::urllib3
-conda-forge::wxpython>=4.0.1
+conda-forge::wxpython>=4.2.0
 conda-forge::xz
 conda-forge::zlib

.conda-envs/macos.txt

@@ -53,6 +53,6 @@ conda-forge::sqlite
 conda-forge::tabulate
 conda-forge::tqdm
 conda-forge::urllib3
-conda-forge::wxpython>=4.0.1
+conda-forge::wxpython>=4.2.0
 conda-forge::xz
 conda-forge::zlib

.conda-envs/windows.txt

@@ -51,6 +51,6 @@ conda-forge::sqlite
 conda-forge::tabulate
 conda-forge::tqdm
 conda-forge::urllib3
-conda-forge::wxpython>=4.0.1
+conda-forge::wxpython>=4.2.0
 conda-forge::xz
 conda-forge::zlib

CHANGELOG.rst

@@ -1,3 +1,94 @@
+DIALS 3.18.1 (2024-03-26)
+=========================
+
+Bugfixes
+--------
+
+- wxPython was inadvertently reverted to 4.1 instead of 4.2 for python 3.9 and 3.10 releases. (`#2636 <https://github.com/dials/dials/issues/2636>`_)
+
+
+DIALS 3.18.0 (2024-02-22)
+=========================
+
+Features
+--------
+
+- ``dials.slice_sequence``: Add ``exclude_images_multiple=`` option to split a scan at calibration images, as an alternative to excluding those images in ``dials.integrate``. (`#2565 <https://github.com/dials/dials/issues/2565>`_)
+- ``dials.index``: Add ``indexing.method=pink_indexer`` for still images, based on `Gevorkov Y et al. pinkIndexer – a universal indexer for pink-beam X-ray and electron diffraction snapshots <https://doi.org/10.1107/S2053273319015559>`_. (`#2604 <https://github.com/dials/dials/issues/2604>`_)
+
+
+Bugfixes
+--------
+
+- ``dials.index``: Correctly output imageset id when ``joint_indexing=False`` and ``max_lattices>1`` (`#2538 <https://github.com/dials/dials/issues/2538>`_)
+- ``dials.index``: Ensure the unindexed_reflections table in the indexer classes is updated during refinement. (`#2545 <https://github.com/dials/dials/issues/2545>`_)
+- When building a new developer installation without CMake, always defer to environmental flags when configuring compilers. This fixes building on MacOS Sonoma; for which the default libtbx configuration failed. (`#2546 <https://github.com/dials/dials/issues/2546>`_)
+- ``dials.cosym``: Ensure correct reindexing of low symmetry data to the correct setting. (`#2548 <https://github.com/dials/dials/issues/2548>`_)
+- ``dials.export``: Unmerged MTZ output now uses gemmi, fixing various longstanding issues. (`#2549 <https://github.com/dials/dials/issues/2549>`_)
+- ``dials.symmetry/dials.cosym``: Avoid crashes when unable to find consistent symmetry during cell reduction routine. (`#2552 <https://github.com/dials/dials/issues/2552>`_)
+- ``dials.sequence_to_stills``: Avoid creating an experiment for an image with no reflections on it. (`#2554 <https://github.com/dials/dials/issues/2554>`_)
+- ``dials.reciprocal_lattice_viewer``: Fix toggling of ids on spotfinding output when viewing multiple files. (`#2566 <https://github.com/dials/dials/issues/2566>`_)
+- ``dials.estimate_resolution``: Fix bug causing fitting failures for isigma, misigma curves. (`#2568 <https://github.com/dials/dials/issues/2568>`_)
+- ``dials.reciprocal_lattice_viewer``: Ensure the correct panel for the beam centre is recorded when loading models. (`#2572 <https://github.com/dials/dials/issues/2572>`_)
+- ``dials.reindex``: When reindexing against a reference, use the best cell when determining lattice symmetry. (`#2573 <https://github.com/dials/dials/issues/2573>`_)
+- ``dials.cosym``: Filter out datasets that can't be mapped through a consistent best cell, rather than exiting. (`#2574 <https://github.com/dials/dials/issues/2574>`_)
+- ``dials.export_bitmaps``: Fix for use of imageset_index parameter on image sets that don't start at image numbers of 1. (`#2591 <https://github.com/dials/dials/issues/2591>`_)
+- Reorganize imports in ``export_mtz.py`` so that its gemmi-less objects can be used by other gemmi-less software. (`#2592 <https://github.com/dials/dials/issues/2592>`_)
+- ``dials.image_viewer``: Fix for stills experiments viewing crash. (`#2594 <https://github.com/dials/dials/issues/2594>`_)
+- ``dials.refine``: Clarify a misleading error message when there are no reflections to refine. (`#2607 <https://github.com/dials/dials/issues/2607>`_)
+- Dependency update for https://github.com/cctbx/dxtbx/pull/697: adds ``natsort`` (`#2611 <https://github.com/dials/dials/issues/2611>`_)
+
+
+Improved Documentation
+----------------------
+
+- ``dials.refine``: Update online documentation. (`#2551 <https://github.com/dials/dials/issues/2551>`_)
+
+
+Misc
+----
+
+- `#2408 <https://github.com/dials/dials/issues/2408>`_, `#2547 <https://github.com/dials/dials/issues/2547>`_, `#2550 <https://github.com/dials/dials/issues/2550>`_, `#2556 <https://github.com/dials/dials/issues/2556>`_, `#2557 <https://github.com/dials/dials/issues/2557>`_, `#2563 <https://github.com/dials/dials/issues/2563>`_, `#2570 <https://github.com/dials/dials/issues/2570>`_, `#2576 <https://github.com/dials/dials/issues/2576>`_, `#2578 <https://github.com/dials/dials/issues/2578>`_, `#2586 <https://github.com/dials/dials/issues/2586>`_, `#2590 <https://github.com/dials/dials/issues/2590>`_, `#2599 <https://github.com/dials/dials/issues/2599>`_, `#2600 <https://github.com/dials/dials/issues/2600>`_, `#2603 <https://github.com/dials/dials/issues/2603>`_
+
+
+Dials 3.18 (2024-02-22)
+=======================
+
+Features
+--------
+
+- ``dials.slice_sequence``: Add ``exclude_images_multiple`` option to split a scan at calibration images, as an alternative to excluding those images in ``dials.integrate``. See #1992 for details. (`#2565 <https://github.com/dials/dials/issues/2565>`_)
+- ``dials.index``: Add ``indexing.method=pink_indexer`` for still images based on Gevorkov Y, Barty A, Brehm W, White TA, Tolstikova A, Wiedorn MO, et al. pinkIndexer – a universal indexer for pink-beam X-ray and electron diffraction snapshots. Acta Cryst A. 2020 Mar 1;76(2):121–31. (`#2604 <https://github.com/dials/dials/issues/2604>`_)
+
+
+Bugfixes
+--------
+
+- ``dials.index``: Correctly output imageset id when joint_indexing=False and max_lattices>1 (`#2538 <https://github.com/dials/dials/issues/2538>`_)
+- ``dials.index``: Ensure the unindexed_reflections table in the indexer classes is updated during refinement. (`#2545 <https://github.com/dials/dials/issues/2545>`_)
+- Always use configured environment when bootstrapping CCTBX. This fixes building on MacOS Sonoma; for which the default libtbx configuration fails. (`#2546 <https://github.com/dials/dials/issues/2546>`_)
+- ``dials.cosym``: Ensure correct reindexing of low symemtry data to the correct setting (`#2548 <https://github.com/dials/dials/issues/2548>`_)
+- ``dials.export``: Unmerged MTZ output now uses gemmi, which fixes various issues (#1099, #1100, #2057, #2379, #2505) (`#2549 <https://github.com/dials/dials/issues/2549>`_)
+- ``dials.refine``: Ensure online documentation is up to date. (`#2551 <https://github.com/dials/dials/issues/2551>`_)
+- ``dials.symmetry/dials.cosym``: Avoid crashes when unable to find consistent symmetry during cell reduction routine. (`#2552 <https://github.com/dials/dials/issues/2552>`_)
+- ``dials.sequence_to_stills``: Avoid creating an experiment for an image with no reflections on it. (`#2554 <https://github.com/dials/dials/issues/2554>`_)
+- ``dials.reciprocal_lattice_viewer``: Fix toggling of ids on spotfinding output when viewing multiple files (`#2566 <https://github.com/dials/dials/issues/2566>`_)
+- ``dials.estimate_resolution``: Fix bug causing fitting failures for isigma, misigma curves (`#2568 <https://github.com/dials/dials/issues/2568>`_)
+- ``dials.reciprocal_lattice_viewer``: ensure the correct panel for the beam centre is recorded when loading models. (`#2572 <https://github.com/dials/dials/issues/2572>`_)
+- ``dials.reindex``: When reindexing against a reference, use the best cell when determining lattice symmetry. (`#2573 <https://github.com/dials/dials/issues/2573>`_)
+- ``dials.cosym``: Filter out datasets that can't be mapped through a consistent best cell, rather than exiting. (`#2574 <https://github.com/dials/dials/issues/2574>`_)
+- ``dials.export_bitmaps``: Fix for use of imageset_index parameter on image sets that don't start at image numbers of 1. (`#2591 <https://github.com/dials/dials/issues/2591>`_)
+- Reorganize imports in ``export_mtz.py`` so that its gemmi-less objects can be used by other gemmi-less software (`#2592 <https://github.com/dials/dials/issues/2592>`_)
+- ``dials.image_viewer``: Fix for stills experiments viewing crash introduced in #2556 (`#2594 <https://github.com/dials/dials/issues/2594>`_)
+- ``dials.refine``: clarify a misleading error message when there are no reflections to refine. (`#2607 <https://github.com/dials/dials/issues/2607>`_)
+
+
+Misc
+----
+
+- `#2408 <https://github.com/dials/dials/issues/2408>`_, `#2547 <https://github.com/dials/dials/issues/2547>`_, `#2550 <https://github.com/dials/dials/issues/2550>`_, `#2556 <https://github.com/dials/dials/issues/2556>`_, `#2557 <https://github.com/dials/dials/issues/2557>`_, `#2563 <https://github.com/dials/dials/issues/2563>`_, `#2570 <https://github.com/dials/dials/issues/2570>`_, `#2576 <https://github.com/dials/dials/issues/2576>`_, `#2578 <https://github.com/dials/dials/issues/2578>`_, `#2586 <https://github.com/dials/dials/issues/2586>`_, `#2590 <https://github.com/dials/dials/issues/2590>`_, `#2599 <https://github.com/dials/dials/issues/2599>`_, `#2600 <https://github.com/dials/dials/issues/2600>`_
+
+
 DIALS 3.17.0 (2023-11-03)
 =========================
 

installer/bootstrap.py

@@ -50,7 +50,7 @@
 allowed_ssh_connections = {}
 concurrent_git_connection_limit = threading.Semaphore(5)
 
-_prebuilt_cctbx_base = "2023"  # Latest Nightly release
+_prebuilt_cctbx_base = "2024"
 
 
 def make_executable(filepath):

newsfragments/1465.bugfix

@@ -0,0 +1 @@
+Add mesa-libgl CDT dependency when building cctbx. This should solve build problems on RHEL8 and other distributions more modern than RHEL7.

newsfragments/2626.misc

@@ -0,0 +1 @@
+Add filter in `TOFSpotFinder._post_process` to remove any reflections outside of the time of flight range

newsfragments/2633.misc

@@ -0,0 +1 @@
+Constrain wxPython version to minimum 4.2.0. This ensures releases are not downgraded.

newsfragments/2634.bugfix

@@ -0,0 +1 @@
+``dials.ssx_index``: Don't combine detector models if individually refined

newsfragments/2638.bugfix

@@ -0,0 +1 @@
+array_family/flex_ext.py: remove nthreads parameter from extract_shoeboxes: is not implemented and is never used anyway

src/dials/algorithms/centroid/__init__.py

@@ -79,7 +79,7 @@ def tof_centroid_px_to_mm_panel(panel, scan, position, variance, sd_error):
 
     pixel_size = panel.get_pixel_size()
     tof = scan.get_property("time_of_flight")  # (usec)
-    frames = [i + 1 for i in range(len(tof))]
+    frames = list(range(len(tof)))
     frame_to_tof = frame_to_tof_interpolator(frames, tof)
 
     if isinstance(position, tuple):

src/dials/algorithms/indexing/ssx/processing.py

@@ -497,9 +497,13 @@ def index(
         method_list,
     )
 
-    # combine detector models if not already
-    if (len(indexed_experiments.detectors())) > 1:
-        combine = CombineWithReference(detector=indexed_experiments[0].detector)
+    # combine detector models if all the same (i.e. haven't been refined in indexing),
+    # to enable use case of joint refinement.
+    detector_0 = indexed_experiments[0].detector if indexed_experiments else None
+    if (len(indexed_experiments.detectors()) > 1) and all(
+        d == detector_0 for d in indexed_experiments.detectors()[1:]
+    ):
+        combine = CombineWithReference(detector=detector_0)
         elist = ExperimentList()
         for expt in indexed_experiments:
             elist.append(combine(expt))

src/dials/algorithms/spot_finding/finder.py

@@ -930,6 +930,20 @@ def _post_process(self, reflections):
 
         reflections = self._correct_centroid_tof(reflections)
 
+        # Filter any reflections outside of the tof range
+        for i, expt in enumerate(self.experiments):
+            _, _, frame = reflections["xyzobs.px.value"].parts()
+            tof_frame_range = (0, len(expt.scan.get_property("time_of_flight")) - 1)
+            if "imageset_id" in reflections:
+                sel_expt = reflections["imageset_id"] == i
+            else:
+                sel_expt = reflections["id"] == i
+
+            sel = sel_expt & (
+                (frame > tof_frame_range[1]) | (frame < tof_frame_range[0])
+            )
+            reflections = reflections.select(~sel)
+
         n_rows = reflections.nrows()
         panel_numbers = flex.size_t(reflections["panel"])
         reflections["L1"] = flex.double(n_rows)

src/dials/array_family/flex_ext.py

@@ -956,7 +956,7 @@ def compute_corrections(self, experiments):
             self["qe"] = qe
         return lp
 
-    def extract_shoeboxes(self, imageset, mask=None, nthreads=1):
+    def extract_shoeboxes(self, imageset, mask=None):
         """
         Helper function to read a load of shoebox data.