Merge pull request #374 from LCOGT/update/sep-astropy

Migrate to no longer use SEP and use astropy Photutils instead.

.github/workflows/build-docker.yml

@@ -0,0 +1,58 @@
+name: Docker Image
+
+concurrency: ${{ github.workflow }}-${{ github.ref }}
+
+on:
+  push:
+    branches:
+      - "*"
+    tags:
+      - "*"
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build-push:
+
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      packages: write
+
+    env:
+      REGISTRY: ghcr.io
+      IMAGE_NAME: ${{ github.repository }}
+
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+
+      - name: Login to Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and also push Dockerimage
+        id: build-and-push
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/e2e.yaml

@@ -45,7 +45,7 @@ jobs:
 
       - name: Test Master Bias Creation
         run: |
-          kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias
+          kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias
       
       - name: Cleanup
         run: |

CHANGES.md

@@ -1,3 +1,7 @@
+1.15.0 (2024-02-14)
+-------------------
+- Migrated photometry extraction to be done by astropy's photutils instead of SEP.
+
 1.14.1 (2024-01-29)
 -------------------
 - Fix for null instrument/instrument size to allow processing to continue
@@ -7,7 +11,7 @@
 - Split worker queues for small and large images
 - Update e2e tests to run on github actions
 
-1.13.1 (2023-10-21)
+1.13.1 (2023-10-31)
 -------------------
 - bugfix to the docker build
 

MANIFEST.in

@@ -13,3 +13,4 @@ prune docs/_build
 prune docs/api
 
 include pyproject.toml
+include pytest.ini

banzai/bpm.py

@@ -39,6 +39,6 @@ def calibration_type(self):
 class SaturatedPixelFlagger(Stage):
     def do_stage(self, image):
         for image_extension in image.ccd_hdus:
-            image_extension.mask[image_extension.data > image_extension.saturate] |= 2
+            image_extension.mask[image_extension.data >= image_extension.saturate] |= 2
 
         return image

banzai/settings.py

@@ -80,7 +80,7 @@
                                   'elp': {'minute': 0, 'hour': 23},
                                   'ogg': {'minute': 0, 'hour': 3}}
 
-ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', 'http://astrometry.lco.gtn/catalog/')
+ASTROMETRY_SERVICE_URL = os.getenv('ASTROMETRY_SERVICE_URL', ' ')
 
 CALIBRATION_FILENAME_FUNCTIONS = {'BIAS': ('banzai.utils.file_utils.config_to_filename',
                                            'banzai.utils.file_utils.ccdsum_to_filename'),

banzai/tests/test_end_to_end.py

@@ -124,7 +124,7 @@ def get_expected_number_of_calibrations(raw_filename_pattern, calibration_type):
             if site in frame['filename'] and instrument in frame['filename']
             and dayobs in frame['filename'] and raw_filename_pattern in frame['filename']
         ]
-        if 'calibration_type.lower()' == 'skyflat':
+        if calibration_type.lower() == 'skyflat':
             # Group by filter
             observed_filters = []
             for frame in raw_frames_for_this_dayobs:
@@ -244,7 +244,7 @@ def test_if_stacked_flat_frame_was_created(self):
 @pytest.mark.e2e
 @pytest.mark.science_files
 class TestScienceFileCreation:
-    @pytest.fixture(autouse=True)
+    @pytest.fixture(autouse=True, scope='class')
     @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect)
     def reduce_science_frames(self, mock_observation_portal):
         run_reduce_individual_frames('e00.fits')
@@ -254,7 +254,7 @@ def test_if_science_frames_were_created(self):
         created_files = []
         for day_obs in DAYS_OBS:
             expected_files += [filename.replace('e00', 'e91')
-                               for filename in TEST_FRAMES['filename']]
+                               for filename in TEST_FRAMES['filename'] if 'e00.fits' in filename]
             created_files += [os.path.basename(filename) for filename in glob(os.path.join(DATA_ROOT, day_obs,
                                                                                            'processed', '*e91*'))]
         assert len(expected_files) > 0
@@ -266,6 +266,6 @@ def test_that_photometric_calibration_succeeded(self):
         for day_obs in DAYS_OBS:
             science_files += [filepath for filepath in glob(os.path.join(DATA_ROOT, day_obs,
                                                                          'processed', '*e91*'))]
-        zeropoints = [fits.open(file)['SCI'].header.get('L1ZP') for file in science_files]
+        zeropoints = [fits.open(filename)['SCI'].header.get('L1ZP') for filename in science_files]
         # check that at least one of our images contains a zeropoint
         assert zeropoints.count(None) != len(zeropoints)

docs/index.rst

@@ -107,20 +107,18 @@ our models are 94% complete for our training data. Pixels flagged as cosmic-ray
 
 Source Detection
 ================
-Source detection uses the "Source Extraction in Python" (SEP; https://github.com/kbarbary/sep).
-This is similar to Source Extractor, but is written purely in Python and Cython. This allows more
-customization.
+Source detection uses the "astropy.photutils" image segmentation.
+This is similar to Source Extractor, but is written purely in Python. This allows more customization than the original SExtractor.
 
 We estimate the background by taking a 3x3 median filter of the image and the doing a 32x32 block
 average of the image.
 
-We use the default match filter for source detection that is provided by SEP.
+We use the default match filter for source detection that is provided in Source Extractor. We include the proper match filter normalization though so the extraction will be slightly different.
 
 We do aperture photometry using an elliptical aperture that is set by 2.5 times the Kron radius. This
 produces approximately the same results as ``FLUX_AUTO`` from SExtractor.
 
-We set the source detection limit at 3 times the global rms of the image. ``MINAREA`` is set to 5,
-the default. This should minimize false detections, but may miss the faintest sources.
+We set the source detection limit at 2.5 times the uncertainty image. ``MINAREA`` is set to 9. This should minimize false detections, but may miss the faintest sources.
 
 To assess the image quality, we estimate the full-width half maximum (FWHM) of the stars in the image. We reject any
 sources that have a FWHM of less than a pixel to ensure that they do not bias our results. The PSFs for LCO are
@@ -132,6 +130,13 @@ estimate of the FWHM. This ensures that we do not underestimate the FWHM when de
 we take the robust standard deviation (see below) to estimate the overall FWHM of the image. This value is recorded
 in the header under the L1FWHM keyword.
 
+Flags are as follows:
+- 1: Source has bad pixels in the image segmentation
+- 2: Object is deblended
+- 4: Source has saturated pixels in the image segmentation
+- 8: Source kron aperture falls off the image
+- 16: Source has cosmic ray pixels in the image segmentation
+
 
 Astrometry
 ==========

helm-chart/banzai/templates/listener.yaml

@@ -98,7 +98,6 @@ spec:
             - "--db-address=$(DB_ADDRESS)"
             - "--broker-url=$(FITS_BROKER)"
             - "--queue-name=$(QUEUE_NAME)"
-            - "--no-file-cache"
           env:
             {{- include "banzai.Env" . | nindent 12 }}
           resources:
@@ -128,7 +127,6 @@ spec:
             - "--db-address=$(DB_ADDRESS)"
             - "--broker-url=$(FITS_BROKER)"
             - "--log-level=info"
-            - "--no-file-cache"
           env:
             {{- include "banzai.Env" . | nindent 12 }}
           resources: