diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1b96cb1d..522b1cd4 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -46,3 +46,7 @@ jobs: - name: Test Master Bias Creation run: | kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias + + - name: Cleanup + run: | + kubectl delete banzai-e2e-test diff --git a/banzai/dark.py b/banzai/dark.py index f053e8ef..17ed5d89 100755 --- a/banzai/dark.py +++ b/banzai/dark.py @@ -2,8 +2,6 @@ from banzai.calibrations import CalibrationStacker, CalibrationUser, CalibrationComparer from banzai.utils import qc from banzai.logs import get_logger -import numpy as np - logger = get_logger() @@ -42,14 +40,9 @@ def calibration_type(self): def apply_master_calibration(self, image, master_calibration_image): master_calibration_image *= image.exptime - temperature_scaling_factor = np.exp(master_calibration_image.dark_temperature_coefficient * \ - (image.measured_ccd_temperature - master_calibration_image.measured_ccd_temperature)) - master_calibration_image *= temperature_scaling_factor - image -= master_calibration_image image.meta['L1IDDARK'] = master_calibration_image.filename, 'ID of dark frame' image.meta['L1STATDA'] = 1, 'Status flag for dark frame correction' - image.meta['DRKTSCAL'] = temperature_scaling_factor, 'Temperature scaling factor applied to dark image' return image diff --git a/banzai/tests/data/test_data.dat b/banzai/tests/data/test_data.dat new file mode 100644 index 00000000..a7a8cda1 --- /dev/null +++ b/banzai/tests/data/test_data.dat @@ -0,0 +1,332 @@ +frameid filename filter +11316611 lsc1m004-fa03-20190402-0010-b00.fits.fz zs +11316617 lsc1m004-fa03-20190402-0011-b00.fits.fz zs +11316623 lsc1m004-fa03-20190402-0012-b00.fits.fz zs +11316625 lsc1m004-fa03-20190402-0013-b00.fits.fz zs +11316631 lsc1m004-fa03-20190402-0014-b00.fits.fz zs +11316635 lsc1m004-fa03-20190402-0015-b00.fits.fz zs +11316643 lsc1m004-fa03-20190402-0016-b00.fits.fz zs +11316648 lsc1m004-fa03-20190402-0017-b00.fits.fz zs +11323171 lsc1m004-fa03-20190402-0298-b00.fits.fz Y +11323183 lsc1m004-fa03-20190402-0299-b00.fits.fz Y +11323202 lsc1m004-fa03-20190402-0300-b00.fits.fz Y +11323220 lsc1m004-fa03-20190402-0301-b00.fits.fz Y +11323233 lsc1m004-fa03-20190402-0302-b00.fits.fz Y +11323245 lsc1m004-fa03-20190402-0303-b00.fits.fz Y +11323256 lsc1m004-fa03-20190402-0304-b00.fits.fz Y +11323268 lsc1m004-fa03-20190402-0305-b00.fits.fz Y +11316793 lsc1m004-fa03-20190402-0018-d00.fits.fz zs +11316854 lsc1m004-fa03-20190402-0019-d00.fits.fz zs +11316904 lsc1m004-fa03-20190402-0020-d00.fits.fz zs +11316997 lsc1m004-fa03-20190402-0021-d00.fits.fz zs +11317038 lsc1m004-fa03-20190402-0022-d00.fits.fz zs +11323485 lsc1m004-fa03-20190402-0315-d00.fits.fz Y +11323588 lsc1m004-fa03-20190402-0316-d00.fits.fz Y +11323681 lsc1m004-fa03-20190402-0317-d00.fits.fz Y +11323749 lsc1m004-fa03-20190402-0318-d00.fits.fz Y +11323792 lsc1m004-fa03-20190402-0319-d00.fits.fz Y +11317268 lsc1m004-fa03-20190402-0033-f00.fits.fz I +11317275 lsc1m004-fa03-20190402-0034-f00.fits.fz I +11317281 lsc1m004-fa03-20190402-0035-f00.fits.fz I +11317289 lsc1m004-fa03-20190402-0036-f00.fits.fz I +11317300 lsc1m004-fa03-20190402-0037-f00.fits.fz I +11322767 lsc1m004-fa03-20190402-0286-f00.fits.fz I +11322784 lsc1m004-fa03-20190402-0287-f00.fits.fz I +11322792 lsc1m004-fa03-20190402-0288-f00.fits.fz I +11322799 lsc1m004-fa03-20190402-0289-f00.fits.fz I +11322809 lsc1m004-fa03-20190402-0290-f00.fits.fz I +11317311 lsc1m004-fa03-20190402-0038-f00.fits.fz V +11317315 lsc1m004-fa03-20190402-0039-f00.fits.fz V +11317325 lsc1m004-fa03-20190402-0040-f00.fits.fz V +11317334 lsc1m004-fa03-20190402-0041-f00.fits.fz V +11317341 lsc1m004-fa03-20190402-0042-f00.fits.fz V +11322616 lsc1m004-fa03-20190402-0276-f00.fits.fz V +11322642 lsc1m004-fa03-20190402-0277-f00.fits.fz V +11322658 lsc1m004-fa03-20190402-0278-f00.fits.fz V +11322676 lsc1m004-fa03-20190402-0279-f00.fits.fz V +11322685 lsc1m004-fa03-20190402-0280-f00.fits.fz V +11318285 lsc1m004-fa03-20190402-0119-e00.fits.fz V +11318300 lsc1m004-fa03-20190402-0120-e00.fits.fz V +11318344 lsc1m004-fa03-20190402-0123-e00.fits.fz I +11318365 lsc1m004-fa03-20190402-0124-e00.fits.fz I +8993484 ogg2m001-fs02-20180630-0003-b00.fits.fz air +8993488 ogg2m001-fs02-20180630-0004-b00.fits.fz air +8993491 ogg2m001-fs02-20180630-0005-b00.fits.fz air +8993493 ogg2m001-fs02-20180630-0006-b00.fits.fz air +8993498 ogg2m001-fs02-20180630-0007-b00.fits.fz air +8993500 ogg2m001-fs02-20180630-0008-b00.fits.fz air +8993505 ogg2m001-fs02-20180630-0009-b00.fits.fz air +8993509 ogg2m001-fs02-20180630-0010-b00.fits.fz air +8993511 ogg2m001-fs02-20180630-0011-b00.fits.fz air +8993515 ogg2m001-fs02-20180630-0012-b00.fits.fz air +8993519 ogg2m001-fs02-20180630-0013-b00.fits.fz air +8993524 ogg2m001-fs02-20180630-0014-b00.fits.fz air +8993526 ogg2m001-fs02-20180630-0015-b00.fits.fz air +8993529 ogg2m001-fs02-20180630-0016-b00.fits.fz air +8993532 ogg2m001-fs02-20180630-0017-b00.fits.fz air +8997180 ogg2m001-fs02-20180630-0183-b00.fits.fz zs +8997183 ogg2m001-fs02-20180630-0184-b00.fits.fz zs +8997190 ogg2m001-fs02-20180630-0185-b00.fits.fz zs +8997192 ogg2m001-fs02-20180630-0186-b00.fits.fz zs +8997194 ogg2m001-fs02-20180630-0187-b00.fits.fz zs +8997196 ogg2m001-fs02-20180630-0188-b00.fits.fz zs +8997198 ogg2m001-fs02-20180630-0189-b00.fits.fz zs +8997200 ogg2m001-fs02-20180630-0190-b00.fits.fz zs +8997202 ogg2m001-fs02-20180630-0191-b00.fits.fz zs +8997204 ogg2m001-fs02-20180630-0192-b00.fits.fz zs +8997206 ogg2m001-fs02-20180630-0193-b00.fits.fz zs +8997208 ogg2m001-fs02-20180630-0194-b00.fits.fz zs +8997211 ogg2m001-fs02-20180630-0195-b00.fits.fz zs +8997213 ogg2m001-fs02-20180630-0196-b00.fits.fz zs +8997216 ogg2m001-fs02-20180630-0197-b00.fits.fz zs +8993616 ogg2m001-fs02-20180630-0018-d00.fits.fz air +8993659 ogg2m001-fs02-20180630-0019-d00.fits.fz air +8993713 ogg2m001-fs02-20180630-0020-d00.fits.fz air +8993796 ogg2m001-fs02-20180630-0021-d00.fits.fz air +8993893 ogg2m001-fs02-20180630-0022-d00.fits.fz air +8997246 ogg2m001-fs02-20180630-0198-d00.fits.fz zs +8997273 ogg2m001-fs02-20180630-0199-d00.fits.fz zs +8997294 ogg2m001-fs02-20180630-0200-d00.fits.fz zs +8997329 ogg2m001-fs02-20180630-0201-d00.fits.fz zs +8997379 ogg2m001-fs02-20180630-0202-d00.fits.fz zs +8996869 ogg2m001-fs02-20180630-0157-f00.fits.fz V +8996875 ogg2m001-fs02-20180630-0158-f00.fits.fz V +8996881 ogg2m001-fs02-20180630-0159-f00.fits.fz V +8996883 ogg2m001-fs02-20180630-0160-f00.fits.fz V +8996885 ogg2m001-fs02-20180630-0161-f00.fits.fz V +8996894 ogg2m001-fs02-20180630-0162-f00.fits.fz ip +8996896 ogg2m001-fs02-20180630-0163-f00.fits.fz ip +8996901 ogg2m001-fs02-20180630-0164-f00.fits.fz ip +8996905 ogg2m001-fs02-20180630-0165-f00.fits.fz ip +8996910 ogg2m001-fs02-20180630-0166-f00.fits.fz ip +8994591 ogg2m001-fs02-20180630-0035-e00.fits.fz V +8994598 ogg2m001-fs02-20180630-0037-e00.fits.fz ip +8995332 ogg2m001-fs02-20180630-0080-e00.fits.fz V +8996719 ogg2m001-fs02-20180630-0138-e00.fits.fz V +9005135 ogg2m001-fs02-20180702-0067-s00.fits.fz V +9044739 ogg0m406-kb27-20180708-0001-b00.fits.fz w +9044743 ogg0m406-kb27-20180708-0002-b00.fits.fz w +9044748 ogg0m406-kb27-20180708-0003-b00.fits.fz w +9044752 ogg0m406-kb27-20180708-0004-b00.fits.fz w +9044757 ogg0m406-kb27-20180708-0005-b00.fits.fz w +9044761 ogg0m406-kb27-20180708-0006-b00.fits.fz w +9044767 ogg0m406-kb27-20180708-0007-b00.fits.fz w +9044772 ogg0m406-kb27-20180708-0008-b00.fits.fz w +9044778 ogg0m406-kb27-20180708-0009-b00.fits.fz w +9044780 ogg0m406-kb27-20180708-0010-b00.fits.fz w +9044786 ogg0m406-kb27-20180708-0011-b00.fits.fz w +9044790 ogg0m406-kb27-20180708-0012-b00.fits.fz w +9044797 ogg0m406-kb27-20180708-0013-b00.fits.fz w +9044805 ogg0m406-kb27-20180708-0014-b00.fits.fz w +9044814 ogg0m406-kb27-20180708-0015-b00.fits.fz w +9044818 ogg0m406-kb27-20180708-0016-b00.fits.fz w +9044824 ogg0m406-kb27-20180708-0017-b00.fits.fz w +9048122 ogg0m406-kb27-20180708-0143-b00.fits.fz zs +9048130 ogg0m406-kb27-20180708-0144-b00.fits.fz zs +9048135 ogg0m406-kb27-20180708-0145-b00.fits.fz zs +9048138 ogg0m406-kb27-20180708-0146-b00.fits.fz zs +9048145 ogg0m406-kb27-20180708-0147-b00.fits.fz zs +9048150 ogg0m406-kb27-20180708-0148-b00.fits.fz zs +9048155 ogg0m406-kb27-20180708-0149-b00.fits.fz zs +9048160 ogg0m406-kb27-20180708-0150-b00.fits.fz zs +9048163 ogg0m406-kb27-20180708-0151-b00.fits.fz zs +9048169 ogg0m406-kb27-20180708-0152-b00.fits.fz zs +9048174 ogg0m406-kb27-20180708-0153-b00.fits.fz zs +9048179 ogg0m406-kb27-20180708-0154-b00.fits.fz zs +9048184 ogg0m406-kb27-20180708-0155-b00.fits.fz zs +9048189 ogg0m406-kb27-20180708-0156-b00.fits.fz zs +9048193 ogg0m406-kb27-20180708-0157-b00.fits.fz zs +9048197 ogg0m406-kb27-20180708-0158-b00.fits.fz zs +9048201 ogg0m406-kb27-20180708-0159-b00.fits.fz zs +9044835 ogg0m406-kb27-20180708-0018-d00.fits.fz w +9044863 ogg0m406-kb27-20180708-0019-d00.fits.fz w +9044894 ogg0m406-kb27-20180708-0020-d00.fits.fz w +9044906 ogg0m406-kb27-20180708-0021-d00.fits.fz w +9044920 ogg0m406-kb27-20180708-0022-d00.fits.fz w +9048213 ogg0m406-kb27-20180708-0160-d00.fits.fz zs +9048232 ogg0m406-kb27-20180708-0161-d00.fits.fz zs +9048258 ogg0m406-kb27-20180708-0162-d00.fits.fz zs +9048296 ogg0m406-kb27-20180708-0163-d00.fits.fz zs +9048311 ogg0m406-kb27-20180708-0164-d00.fits.fz zs +9047791 ogg0m406-kb27-20180708-0123-f00.fits.fz rp +9047809 ogg0m406-kb27-20180708-0124-f00.fits.fz rp +9047825 ogg0m406-kb27-20180708-0125-f00.fits.fz rp +9047839 ogg0m406-kb27-20180708-0126-f00.fits.fz rp +9047847 ogg0m406-kb27-20180708-0127-f00.fits.fz rp +9047909 ogg0m406-kb27-20180708-0133-f00.fits.fz ip +9047917 ogg0m406-kb27-20180708-0134-f00.fits.fz ip +9047925 ogg0m406-kb27-20180708-0135-f00.fits.fz ip +9047933 ogg0m406-kb27-20180708-0136-f00.fits.fz ip +9047943 ogg0m406-kb27-20180708-0137-f00.fits.fz ip +9046502 ogg0m406-kb27-20180708-0072-e00.fits.fz rp +9046515 ogg0m406-kb27-20180708-0073-e00.fits.fz rp +9046529 ogg0m406-kb27-20180708-0074-e00.fits.fz ip +9046539 ogg0m406-kb27-20180708-0075-e00.fits.fz ip +9011552 ogg0m406-kb27-20180703-0065-e00.fits.fz rp +9011558 ogg0m406-kb27-20180703-0066-e00.fits.fz ip +11896333 cpt1m012-fa06-20190609-0038-f00.fits.fz R +11896336 cpt1m012-fa06-20190609-0039-f00.fits.fz R +11896340 cpt1m012-fa06-20190609-0040-f00.fits.fz R +11896342 cpt1m012-fa06-20190609-0041-f00.fits.fz R +11896345 cpt1m012-fa06-20190609-0042-f00.fits.fz R +11900872 cpt1m012-fa06-20190609-0120-b00.fits.fz rp +11900878 cpt1m012-fa06-20190609-0121-b00.fits.fz rp +11900883 cpt1m012-fa06-20190609-0122-b00.fits.fz rp +11900888 cpt1m012-fa06-20190609-0123-b00.fits.fz rp +11900896 cpt1m012-fa06-20190609-0124-b00.fits.fz rp +11900897 cpt1m012-fa06-20190609-0125-b00.fits.fz rp +11900898 cpt1m012-fa06-20190609-0126-b00.fits.fz rp +11900904 cpt1m012-fa06-20190609-0127-b00.fits.fz rp +11900909 cpt1m012-fa06-20190609-0128-b00.fits.fz rp +11900911 cpt1m012-fa06-20190609-0129-b00.fits.fz rp +11900914 cpt1m012-fa06-20190609-0130-b00.fits.fz rp +11900917 cpt1m012-fa06-20190609-0131-b00.fits.fz rp +11900920 cpt1m012-fa06-20190609-0132-b00.fits.fz rp +11900924 cpt1m012-fa06-20190609-0133-b00.fits.fz rp +11900927 cpt1m012-fa06-20190609-0134-b00.fits.fz rp +11901029 cpt1m012-fa06-20190609-0135-d00.fits.fz rp +11901113 cpt1m012-fa06-20190609-0136-d00.fits.fz rp +11901214 cpt1m012-fa06-20190609-0137-d00.fits.fz rp +11901308 cpt1m012-fa06-20190609-0138-d00.fits.fz rp +11901401 cpt1m012-fa06-20190609-0139-d00.fits.fz rp +11903076 cpt1m012-fa06-20190610-0038-f00.fits.fz R +11903080 cpt1m012-fa06-20190610-0039-f00.fits.fz R +11903084 cpt1m012-fa06-20190610-0040-f00.fits.fz R +11903086 cpt1m012-fa06-20190610-0041-f00.fits.fz R +11903089 cpt1m012-fa06-20190610-0042-f00.fits.fz R +11910164 cpt1m012-fa06-20190610-0101-b00.fits.fz V +11910170 cpt1m012-fa06-20190610-0102-b00.fits.fz V +11910178 cpt1m012-fa06-20190610-0103-b00.fits.fz V +11910180 cpt1m012-fa06-20190610-0104-b00.fits.fz V +11910185 cpt1m012-fa06-20190610-0105-b00.fits.fz V +11910188 cpt1m012-fa06-20190610-0106-b00.fits.fz V +11910192 cpt1m012-fa06-20190610-0107-b00.fits.fz V +11910198 cpt1m012-fa06-20190610-0108-b00.fits.fz V +11910200 cpt1m012-fa06-20190610-0109-b00.fits.fz V +11910203 cpt1m012-fa06-20190610-0110-b00.fits.fz V +11910209 cpt1m012-fa06-20190610-0111-b00.fits.fz V +11910213 cpt1m012-fa06-20190610-0112-b00.fits.fz V +11910212 cpt1m012-fa06-20190610-0113-b00.fits.fz V +11910215 cpt1m012-fa06-20190610-0114-b00.fits.fz V +11910218 cpt1m012-fa06-20190610-0115-b00.fits.fz V +11910300 cpt1m012-fa06-20190610-0116-d00.fits.fz V +11910419 cpt1m012-fa06-20190610-0117-d00.fits.fz V +11910485 cpt1m012-fa06-20190610-0118-d00.fits.fz V +11910579 cpt1m012-fa06-20190610-0119-d00.fits.fz V +11910656 cpt1m012-fa06-20190610-0120-d00.fits.fz V +11937481 cpt1m012-fa06-20190614-0067-e00.fits.fz R +11937476 cpt1m012-fa06-20190614-0066-e00.fits.fz R +11937473 cpt1m012-fa06-20190614-0065-e00.fits.fz R +11937461 cpt1m012-fa06-20190614-0064-e00.fits.fz R +11937454 cpt1m012-fa06-20190614-0063-e00.fits.fz R +11937445 cpt1m012-fa06-20190614-0062-e00.fits.fz R +11937432 cpt1m012-fa06-20190614-0061-e00.fits.fz R +11937416 cpt1m012-fa06-20190614-0060-e00.fits.fz R +11937407 cpt1m012-fa06-20190614-0059-e00.fits.fz R +11937400 cpt1m012-fa06-20190614-0058-e00.fits.fz R +33347266 tst1m0XX-ep01-20200324-0001-b00.fits.fz rp*diffuser +33347267 tst1m0XX-ep01-20200324-0002-b00.fits.fz rp*diffuser +33347268 tst1m0XX-ep01-20200324-0003-b00.fits.fz rp*diffuser +33347269 tst1m0XX-ep01-20200324-0004-b00.fits.fz rp*diffuser +33347270 tst1m0XX-ep01-20200324-0005-b00.fits.fz rp*diffuser +33347271 tst1m0XX-ep01-20200324-0006-b00.fits.fz rp*diffuser +33347272 tst1m0XX-ep01-20200324-0007-b00.fits.fz rp*diffuser +33347273 tst1m0XX-ep01-20200324-0008-b00.fits.fz rp*diffuser +33347274 tst1m0XX-ep01-20200324-0009-b00.fits.fz rp*diffuser +33347275 tst1m0XX-ep01-20200324-0010-b00.fits.fz rp*diffuser +33347276 tst1m0XX-ep02-20200324-0001-b00.fits.fz gp*diffuser +33347277 tst1m0XX-ep02-20200324-0002-b00.fits.fz gp*diffuser +33347278 tst1m0XX-ep02-20200324-0003-b00.fits.fz gp*diffuser +33347279 tst1m0XX-ep02-20200324-0004-b00.fits.fz gp*diffuser +33347280 tst1m0XX-ep02-20200324-0005-b00.fits.fz gp*diffuser +33347281 tst1m0XX-ep02-20200324-0006-b00.fits.fz gp*diffuser +33347282 tst1m0XX-ep02-20200324-0007-b00.fits.fz gp*diffuser +33347283 tst1m0XX-ep02-20200324-0008-b00.fits.fz gp*diffuser +33347284 tst1m0XX-ep02-20200324-0009-b00.fits.fz gp*diffuser +33347285 tst1m0XX-ep02-20200324-0010-b00.fits.fz gp*diffuser +33347286 tst1m0XX-ep03-20200324-0001-b00.fits.fz zp*diffuser +33347287 tst1m0XX-ep03-20200324-0002-b00.fits.fz zp*diffuser +33347288 tst1m0XX-ep03-20200324-0003-b00.fits.fz zp*diffuser +33347289 tst1m0XX-ep03-20200324-0004-b00.fits.fz zp*diffuser +33347290 tst1m0XX-ep03-20200324-0005-b00.fits.fz zp*diffuser +33347291 tst1m0XX-ep03-20200324-0006-b00.fits.fz zp*diffuser +33347292 tst1m0XX-ep03-20200324-0007-b00.fits.fz zp*diffuser +33347293 tst1m0XX-ep03-20200324-0008-b00.fits.fz zp*diffuser +33347294 tst1m0XX-ep03-20200324-0009-b00.fits.fz zp*diffuser +33347295 tst1m0XX-ep03-20200324-0010-b00.fits.fz zp*diffuser +33347296 tst1m0XX-ep04-20200324-0001-b00.fits.fz ip*diffuser +33347297 tst1m0XX-ep04-20200324-0002-b00.fits.fz ip*diffuser +33347298 tst1m0XX-ep04-20200324-0003-b00.fits.fz ip*diffuser +33347299 tst1m0XX-ep04-20200324-0004-b00.fits.fz ip*diffuser +33347300 tst1m0XX-ep04-20200324-0005-b00.fits.fz ip*diffuser +33347301 tst1m0XX-ep04-20200324-0006-b00.fits.fz ip*diffuser +33347302 tst1m0XX-ep04-20200324-0007-b00.fits.fz ip*diffuser +33347303 tst1m0XX-ep04-20200324-0008-b00.fits.fz ip*diffuser +33347304 tst1m0XX-ep04-20200324-0009-b00.fits.fz ip*diffuser +33347305 tst1m0XX-ep04-20200324-0010-b00.fits.fz ip*diffuser +33347306 tst1m0XX-ep01-20200324-0011-d00.fits.fz rp*diffuser +33347307 tst1m0XX-ep01-20200324-0012-d00.fits.fz rp*diffuser +33347308 tst1m0XX-ep01-20200324-0013-d00.fits.fz rp*diffuser +33347309 tst1m0XX-ep01-20200324-0014-d00.fits.fz rp*diffuser +33347310 tst1m0XX-ep01-20200324-0015-d00.fits.fz rp*diffuser +33347311 tst1m0XX-ep01-20200324-0016-d00.fits.fz rp*diffuser +33347312 tst1m0XX-ep01-20200324-0017-d00.fits.fz rp*diffuser +33347313 tst1m0XX-ep01-20200324-0018-d00.fits.fz rp*diffuser +33347314 tst1m0XX-ep01-20200324-0019-d00.fits.fz rp*diffuser +33347315 tst1m0XX-ep01-20200324-0020-d00.fits.fz rp*diffuser +33347316 tst1m0XX-ep02-20200324-0011-d00.fits.fz gp*diffuser +33347317 tst1m0XX-ep02-20200324-0012-d00.fits.fz gp*diffuser +33347318 tst1m0XX-ep02-20200324-0013-d00.fits.fz gp*diffuser +33347319 tst1m0XX-ep02-20200324-0014-d00.fits.fz gp*diffuser +33347320 tst1m0XX-ep02-20200324-0015-d00.fits.fz gp*diffuser +33347321 tst1m0XX-ep02-20200324-0016-d00.fits.fz gp*diffuser +33347322 tst1m0XX-ep02-20200324-0017-d00.fits.fz gp*diffuser +33347323 tst1m0XX-ep02-20200324-0018-d00.fits.fz gp*diffuser +33347324 tst1m0XX-ep02-20200324-0019-d00.fits.fz gp*diffuser +33347325 tst1m0XX-ep02-20200324-0020-d00.fits.fz gp*diffuser +33347326 tst1m0XX-ep03-20200324-0011-d00.fits.fz zp*diffuser +33347327 tst1m0XX-ep03-20200324-0012-d00.fits.fz zp*diffuser +33347328 tst1m0XX-ep03-20200324-0013-d00.fits.fz zp*diffuser +33347329 tst1m0XX-ep03-20200324-0014-d00.fits.fz zp*diffuser +33347330 tst1m0XX-ep03-20200324-0015-d00.fits.fz zp*diffuser +33347331 tst1m0XX-ep03-20200324-0016-d00.fits.fz zp*diffuser +33347332 tst1m0XX-ep03-20200324-0017-d00.fits.fz zp*diffuser +33347333 tst1m0XX-ep03-20200324-0018-d00.fits.fz zp*diffuser +33347334 tst1m0XX-ep03-20200324-0019-d00.fits.fz zp*diffuser +33347335 tst1m0XX-ep03-20200324-0020-d00.fits.fz zp*diffuser +33347336 tst1m0XX-ep04-20200324-0011-d00.fits.fz ip*diffuser +33347337 tst1m0XX-ep04-20200324-0012-d00.fits.fz ip*diffuser +33347338 tst1m0XX-ep04-20200324-0013-d00.fits.fz ip*diffuser +33347339 tst1m0XX-ep04-20200324-0014-d00.fits.fz ip*diffuser +33347340 tst1m0XX-ep04-20200324-0015-d00.fits.fz ip*diffuser +33347341 tst1m0XX-ep04-20200324-0016-d00.fits.fz ip*diffuser +33347342 tst1m0XX-ep04-20200324-0017-d00.fits.fz ip*diffuser +33347343 tst1m0XX-ep04-20200324-0018-d00.fits.fz ip*diffuser +33347344 tst1m0XX-ep04-20200324-0019-d00.fits.fz ip*diffuser +33347345 tst1m0XX-ep04-20200324-0020-d00.fits.fz ip*diffuser +56016937 ogg0m404-sq30-20221123-0037-b00.fits.fz opaque +56016931 ogg0m404-sq30-20221123-0036-b00.fits.fz opaque +56016928 ogg0m404-sq30-20221123-0035-b00.fits.fz opaque +56016925 ogg0m404-sq30-20221123-0034-b00.fits.fz opaque +56016920 ogg0m404-sq30-20221123-0033-b00.fits.fz opaque +56016916 ogg0m404-sq30-20221123-0032-b00.fits.fz opaque +56016912 ogg0m404-sq30-20221123-0031-b00.fits.fz opaque +56016909 ogg0m404-sq30-20221123-0030-b00.fits.fz opaque +56016906 ogg0m404-sq30-20221123-0029-b00.fits.fz opaque +56016903 ogg0m404-sq30-20221123-0028-b00.fits.fz opaque +56016899 ogg0m404-sq30-20221123-0027-b00.fits.fz opaque +56016898 ogg0m404-sq30-20221123-0026-b00.fits.fz opaque +56016896 ogg0m404-sq30-20221123-0025-b00.fits.fz opaque +56016891 ogg0m404-sq30-20221123-0024-b00.fits.fz opaque +56016889 ogg0m404-sq30-20221123-0023-b00.fits.fz opaque +56017384 ogg0m404-sq30-20221123-0042-d00.fits.fz opaque +56017294 ogg0m404-sq30-20221123-0041-d00.fits.fz opaque +56017205 ogg0m404-sq30-20221123-0040-d00.fits.fz opaque +56017136 ogg0m404-sq30-20221123-0039-d00.fits.fz opaque +56017044 ogg0m404-sq30-20221123-0038-d00.fits.fz opaque +56018344 ogg0m404-sq30-20221123-0067-f00.fits.fz rp +56018340 ogg0m404-sq30-20221123-0066-f00.fits.fz rp +56018336 ogg0m404-sq30-20221123-0065-f00.fits.fz rp +56018331 ogg0m404-sq30-20221123-0064-f00.fits.fz rp +56018328 ogg0m404-sq30-20221123-0063-f00.fits.fz rp +56127852 ogg0m404-sq30-20221126-0098-e00.fits.fz rp diff --git a/banzai/tests/data/test_precals.dat b/banzai/tests/data/test_precals.dat new file mode 100644 index 00000000..68ee3a75 --- /dev/null +++ b/banzai/tests/data/test_precals.dat @@ -0,0 +1,11 @@ +frameid filename binning mode obstype dateobs site instrument +9741441 bpm_lsc_fa03_20181001_bin1x1.fits.fz '[1, 1]' default BPM lsc fa03 +3519536 bpm_ogg_fs02_20140325_bin2x2.fits.fz '[2, 2]' default BPM ogg fs02 +8831324 bpm_ogg_kb27_20180605_bin1x1.fits.fz '[1, 1]' default BPM ogg kb27 +13261910 bpm-cpt-fa06-central2k-20190617.fits.fz '[2, 2]' central2k BPM cpt fa06 +33347346 tst1m0XX-ep01-20200710-bpm-1.fits.fz '[1, 1]' default BPM tst ep01 +33347347 tst1m0XX-ep02-20200710-bpm-1.fits.fz '[1, 1]' default BPM tst ep02 +33347348 tst1m0XX-ep03-20200710-bpm-1.fits.fz '[1, 1]' default BPM tst ep03 +33347349 tst1m0XX-ep04-20200710-bpm-1.fits.fz '[1, 1]' default BPM tst ep04 +58186456 ogg0m404-sq30-20221123-readnoise-default-e2e.fits.fz '[1, 1]' default READNOISE ogg sq30 +56389948 ogg0m404-sq30-20221206-bpm-default.fits.fz '[1, 1]' default BPM ogg sq30 diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index fac1ccec..0ec065fe 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -13,33 +13,7 @@ spec: securityContext: fsGroup: 10000 - initContainers: - # When the Pod is initialized, copy all files within the container at path - # /archive/engineering into the empty data volume mounted at /data - - name: banzai-data - image: docker.lco.global/banzai-e2e-data:1.4.2 - imagePullPolicy: IfNotPresent - securityContext: - runAsUser: 10087 - runAsGroup: 10000 - volumeMounts: - - name: banzai-data - mountPath: /data - readOnly: false - command: - - /bin/cp - - -a - - /archive/engineering - - /data/ - resources: - requests: - cpu: 0.1 - memory: 256M - limits: - cpu: 1 - memory: 1Gi - - containers: + containers: - name: banzai-redis image: redis:5.0.3 imagePullPolicy: IfNotPresent @@ -141,7 +115,7 @@ spec: memory: 6Gi limits: cpu: 8 - memory: 8Gi + memory: 10Gi - name: banzai-celery-beat image: banzai:test-latest imagePullPolicy: IfNotPresent diff --git a/banzai/tests/test_end_to_end.py b/banzai/tests/test_end_to_end.py index 33357930..43ec4a8e 100644 --- a/banzai/tests/test_end_to_end.py +++ b/banzai/tests/test_end_to_end.py @@ -13,11 +13,11 @@ from types import ModuleType from banzai.celery import app, schedule_calibration_stacking from banzai.dbs import get_session, CalibrationImage, get_timezone, populate_instrument_tables -from banzai.dbs import mark_frame -from banzai.utils import fits_utils, file_utils +from banzai.dbs import mark_frame, query_for_instrument +from banzai.utils import file_utils from banzai.main import add_super_calibration from banzai.tests.utils import FakeResponse, get_min_and_max_dates, FakeContext -from astropy.io import fits +from astropy.io import fits, ascii import pkg_resources from banzai.logs import get_logger @@ -28,15 +28,18 @@ app.conf.update(CELERY_TASK_ALWAYS_EAGER=True) +TEST_PACKAGE = 'banzai.tests' +TEST_FRAMES = ascii.read(pkg_resources.resource_filename(TEST_PACKAGE, 'data/test_data.dat')) + +PRECAL_FRAMES = ascii.read(pkg_resources.resource_filename(TEST_PACKAGE, 'data/test_precal.dat')) + DATA_ROOT = os.path.join(os.sep, 'archive', 'engineering') -SITES = [os.path.basename(site_path) for site_path in glob(os.path.join(DATA_ROOT, '???'))] -INSTRUMENTS = [os.path.join(site, os.path.basename(instrument_path)) for site in SITES - for instrument_path in glob(os.path.join(os.path.join(DATA_ROOT, site, '*')))] +# Use the LCO filenaming convention to infer the sites +SITES = set([frame[:3] for frame in TEST_FRAMES['filename']]) +INSTRUMENTS = set([os.path.join(frame[:3], frame.split('-')[1]) for frame in TEST_FRAMES['filename']]) -DAYS_OBS = [os.path.join(instrument, os.path.basename(dayobs_path)) for instrument in INSTRUMENTS - for dayobs_path in glob(os.path.join(DATA_ROOT, instrument, '20*'))] +DAYS_OBS = set([os.path.join(frame[:3], frame.split('-')[1], frame.split('-')[2]) for frame in TEST_FRAMES['filename']]) -TEST_PACKAGE = 'banzai.tests' CONFIGDB_FILENAME = pkg_resources.resource_filename(TEST_PACKAGE, 'data/configdb_example.json') @@ -62,14 +65,14 @@ def celery_join(): break -def run_reduce_individual_frames(raw_filenames): - logger.info('Reducing individual frames for filenames: {filenames}'.format(filenames=raw_filenames)) - for day_obs in DAYS_OBS: - raw_path = os.path.join(DATA_ROOT, day_obs, 'raw') - for filename in glob(os.path.join(raw_path, raw_filenames)): - file_utils.post_to_archive_queue(filename, os.getenv('FITS_BROKER'), exchange_name=os.getenv('FITS_EXCHANGE')) +def run_reduce_individual_frames(filename_pattern): + logger.info('Reducing individual frames for filenames: {filenames}'.format(filenames=filename_pattern)) + for frame in TEST_FRAMES: + if filename_pattern in frame['filename']: + file_utils.post_to_archive_queue(frame['filename'], frame['frameid'], os.getenv('FITS_BROKER'), + exchange_name=os.getenv('FITS_EXCHANGE')) celery_join() - logger.info('Finished reducing individual frames for filenames: {filenames}'.format(filenames=raw_filenames)) + logger.info('Finished reducing individual frames for filenames: {filenames}'.format(filenames=filename_pattern)) def stack_calibrations(frame_type): @@ -102,30 +105,34 @@ def mark_frames_as_good(raw_filenames): logger.info('Finished marking frames as good for filenames: {filenames}'.format(filenames=raw_filenames)) -def get_expected_number_of_calibrations(raw_filenames, calibration_type): +def get_expected_number_of_calibrations(raw_filename_pattern, calibration_type): context = FakeContext() context.db_address = os.environ['DB_ADDRESS'] number_of_stacks_that_should_have_been_created = 0 for day_obs in DAYS_OBS: - raw_filenames_for_this_dayobs = glob(os.path.join(DATA_ROOT, day_obs, 'raw', raw_filenames)) - if calibration_type.lower() == 'skyflat': + site, instrument, dayobs = day_obs.split('/') + raw_frames_for_this_dayobs = [ + frame for frame in TEST_FRAMES + if site in frame['filename'] and instrument in frame['filename'] + and dayobs in frame['filename'] and raw_filename_pattern in frame['filename'] + ] + if 'calibration_type.lower()' == 'skyflat': # Group by filter observed_filters = [] - for raw_filename in raw_filenames_for_this_dayobs: - skyflat_hdu, skyflat_filename, frame_id = fits_utils.open_fits_file({'path': raw_filename}, context) - observed_filters.append(skyflat_hdu[0].header.get('FILTER')) + for frame in raw_frames_for_this_dayobs: + observed_filters.append(frame['filter']) observed_filters = set(observed_filters) number_of_stacks_that_should_have_been_created += len(observed_filters) else: # Just one calibration per night - if len(raw_filenames_for_this_dayobs) > 0: + if len(raw_frames_for_this_dayobs) > 0: number_of_stacks_that_should_have_been_created += 1 return number_of_stacks_that_should_have_been_created -def run_check_if_stacked_calibrations_were_created(raw_filenames, calibration_type): +def run_check_if_stacked_calibrations_were_created(raw_file_pattern, calibration_type): created_stacked_calibrations = [] - number_of_stacks_that_should_have_been_created = get_expected_number_of_calibrations(raw_filenames, calibration_type) + number_of_stacks_that_should_have_been_created = get_expected_number_of_calibrations(raw_file_pattern, calibration_type) for day_obs in DAYS_OBS: created_stacked_calibrations += glob(os.path.join(DATA_ROOT, day_obs, 'processed', '*' + calibration_type.lower() + '*.fits*')) @@ -133,8 +140,9 @@ def run_check_if_stacked_calibrations_were_created(raw_filenames, calibration_ty assert len(created_stacked_calibrations) == number_of_stacks_that_should_have_been_created -def run_check_if_stacked_calibrations_are_in_db(raw_filenames, calibration_type): - number_of_stacks_that_should_have_been_created = get_expected_number_of_calibrations(raw_filenames, calibration_type) +def run_check_if_stacked_calibrations_are_in_db(raw_file_pattern, calibration_type): + number_of_stacks_that_should_have_been_created = get_expected_number_of_calibrations(raw_file_pattern, + calibration_type) with get_session(os.environ['DB_ADDRESS']) as db_session: calibrations_in_db = db_session.query(CalibrationImage).filter(CalibrationImage.type == calibration_type) calibrations_in_db = calibrations_in_db.filter(CalibrationImage.is_master).all() @@ -156,19 +164,28 @@ def observation_portal_side_effect(*args, **kwargs): # Note this is complicated by the fact that things are running as celery tasks. @pytest.mark.e2e @pytest.fixture(scope='module') -@mock.patch('banzai.main.argparse.ArgumentParser.parse_args') -@mock.patch('banzai.main.file_utils.post_to_ingester', return_value={'frameid': None}) @mock.patch('banzai.dbs.requests.get', return_value=FakeResponse(CONFIGDB_FILENAME)) def init(configdb, mock_ingester, mock_args): os.system(f'banzai_create_db --db-address={os.environ["DB_ADDRESS"]}') populate_instrument_tables(db_address=os.environ["DB_ADDRESS"], configdb_address='http://fakeconfigdb') - for instrument in INSTRUMENTS: - for bpm_filepath in glob(os.path.join(DATA_ROOT, instrument, 'bpm/*bpm*')): - mock_args.return_value = argparse.Namespace(filepath=bpm_filepath, db_address=os.environ['DB_ADDRESS'], log_level='debug') - add_super_calibration() - for noise_map_filepath in glob(os.path.join(DATA_ROOT, instrument, 'readnoise/*readnoise*')): - mock_args.return_value = argparse.Namespace(filepath=noise_map_filepath, db_address=os.environ['DB_ADDRESS'], log_level='debug') - add_super_calibration() + + for frame in PRECAL_FRAMES: + instrument = query_for_instrument(camera=frame['instrument'], + site=frame['site'], + db_address=os.environ['DB_ADDRESS']) + calimage = CalibrationImage( + type=frame['obstype'], + filename=frame['filename'], + frameid=frame['frameid'], + dateobs=frame['dateobs'], + datecreated='2023-11-19', + instrument_id=instrument.id, + is_master=True, is_bad=False, + attributes={'binning': frame['binning'], 'configuration_mode': frame['mode']} + ) + with get_session(os.environ['DB_ADDRESS']) as db_session: + db_session.add(calimage) + db_session.commit() @pytest.mark.e2e @@ -177,13 +194,13 @@ class TestMasterBiasCreation: @pytest.fixture(autouse=True) @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect) def stack_bias_frames(self, mock_observation_portal, init): - run_reduce_individual_frames('*b00.fits*') + run_reduce_individual_frames('b00.fits') mark_frames_as_good('*b91.fits*') stack_calibrations('bias') def test_if_stacked_bias_frame_was_created(self): - run_check_if_stacked_calibrations_were_created('*b00.fits*', 'bias') - run_check_if_stacked_calibrations_are_in_db('*b00.fits*', 'BIAS') + run_check_if_stacked_calibrations_were_created('b00.fits', 'bias') + run_check_if_stacked_calibrations_are_in_db('b00.fits', 'BIAS') @pytest.mark.e2e @@ -192,13 +209,13 @@ class TestMasterDarkCreation: @pytest.fixture(autouse=True) @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect) def stack_dark_frames(self, mock_observation_portal): - run_reduce_individual_frames('*d00.fits*') + run_reduce_individual_frames('d00.fits') mark_frames_as_good('*d91.fits*') stack_calibrations('dark') def test_if_stacked_dark_frame_was_created(self): - run_check_if_stacked_calibrations_were_created('*d00.fits*', 'dark') - run_check_if_stacked_calibrations_are_in_db('*d00.fits*', 'DARK') + run_check_if_stacked_calibrations_were_created('d00.fits', 'dark') + run_check_if_stacked_calibrations_are_in_db('d00.fits', 'DARK') @pytest.mark.e2e @@ -207,13 +224,13 @@ class TestMasterFlatCreation: @pytest.fixture(autouse=True) @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect) def stack_flat_frames(self, mock_observation_portal): - run_reduce_individual_frames('*f00.fits*') + run_reduce_individual_frames('f00.fits') mark_frames_as_good('*f91.fits*') stack_calibrations('skyflat') def test_if_stacked_flat_frame_was_created(self): - run_check_if_stacked_calibrations_were_created('*f00.fits*', 'skyflat') - run_check_if_stacked_calibrations_are_in_db('*f00.fits*', 'SKYFLAT') + run_check_if_stacked_calibrations_were_created('f00.fits', 'skyflat') + run_check_if_stacked_calibrations_are_in_db('f00.fits', 'SKYFLAT') @pytest.mark.e2e @@ -222,14 +239,14 @@ class TestScienceFileCreation: @pytest.fixture(autouse=True) @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect) def reduce_science_frames(self, mock_observation_portal): - run_reduce_individual_frames('*e00.fits*') + run_reduce_individual_frames('e00.fits') def test_if_science_frames_were_created(self): expected_files = [] created_files = [] for day_obs in DAYS_OBS: - expected_files += [os.path.basename(filename).replace('e00', 'e91') - for filename in glob(os.path.join(DATA_ROOT, day_obs, 'raw', '*e00*'))] + expected_files += [filename.replace('e00', 'e91') + for filename in TEST_FRAMES['filename']] created_files += [os.path.basename(filename) for filename in glob(os.path.join(DATA_ROOT, day_obs, 'processed', '*e91*'))] assert len(expected_files) > 0 diff --git a/banzai/utils/file_utils.py b/banzai/utils/file_utils.py index 610a5196..bc105f5e 100755 --- a/banzai/utils/file_utils.py +++ b/banzai/utils/file_utils.py @@ -11,11 +11,11 @@ logger = get_logger() -def post_to_archive_queue(image_path, broker_url, exchange_name='fits_files'): +def post_to_archive_queue(filename, frameid, broker_url, exchange_name='fits_files'): exchange = Exchange(exchange_name, type='fanout') with Connection(broker_url) as conn: producer = conn.Producer(exchange=exchange) - producer.publish({'path': image_path}) + producer.publish({'filename': filename, 'frameid': frameid}) producer.release()