Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix arrays not being masked correctly in flatfield calculator #1149

Merged
merged 6 commits into from
Aug 29, 2023
Merged

Fix arrays not being masked correctly in flatfield calculator #1149

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ba673a0
Fix arrays not being masked correctly in flatfield calculator
maxnoe Aug 24, 2023
7b8884f
Simplify code a bit
maxnoe Aug 24, 2023
2bdfe4b
Update mask treatment of pixels with NaN median values
FrancaCassol Aug 25, 2023
0933e9b
make all masked pixels as outliers
FrancaCassol Aug 28, 2023
8c23513
Set to zero time corrections of unusable pixels
FrancaCassol Aug 29, 2023
d91e876
Set to Nan values of masked pixels
FrancaCassol Aug 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions lstchain/calib/camera/flatfield.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,13 @@ def calculate_relative_gain_results(
axis=0,
)

# mask pixels without defined statistical values (mainly due to hardware problems)
pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean))
pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median))
pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std))

unused_values = np.abs(masked_trace_integral - pixel_mean) > (max_sigma * pixel_std)

# only warn for values discard in the sigma clipping, not those from before
outliers = unused_values & (~masked_trace_integral.mask)
check_outlier_mask(outliers, self.log, "flatfield")
Expand All @@ -356,26 +362,20 @@ def calculate_relative_gain_results(
charge_median_outliers = (
np.logical_or(charge_deviation < self.charge_median_cut_outliers[0] * median_of_pixel_median[:,np.newaxis],
charge_deviation > self.charge_median_cut_outliers[1] * median_of_pixel_median[:,np.newaxis]))



# outliers from standard deviation
deviation = pixel_std - median_of_pixel_std[:, np.newaxis]
charge_std_outliers = (
np.logical_or(deviation < self.charge_std_cut_outliers[0] * std_of_pixel_std[:, np.newaxis],
deviation > self.charge_std_cut_outliers[1] * std_of_pixel_std[:, np.newaxis]))

# mask pixels with NaN mean, due to missing statistics
pixels_without_stat = np.where(np.isnan(pixel_mean)==True)
charge_median_outliers[pixels_without_stat] = True
charge_std_outliers[pixels_without_stat] = True

return {
'relative_gain_median': np.ma.getdata(np.ma.median(relative_gain_event, axis=0)),
'relative_gain_mean': np.ma.getdata(np.ma.mean(relative_gain_event, axis=0)),
'relative_gain_std': np.ma.getdata(np.ma.std(relative_gain_event, axis=0)),
'charge_median': np.ma.getdata(pixel_median),
'charge_mean': np.ma.getdata(pixel_mean),
'charge_std': np.ma.getdata(pixel_std),
'charge_std_outliers': np.ma.getdata(charge_std_outliers),
'charge_median_outliers': np.ma.getdata(charge_median_outliers),
'charge_std_outliers': charge_std_outliers.filled(True),
'charge_median_outliers': charge_median_outliers.filled(True)
}
17 changes: 9 additions & 8 deletions lstchain/calib/camera/pedestals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,13 @@ def calculate_pedestal_results(self, trace_integral, masked_pixels_of_sample):
axis=0,
)

# mask pixels without defined statistical values (mainly due to hardware problems)
pixel_mean = np.ma.array(pixel_mean, mask=np.isnan(pixel_mean))
pixel_median = np.ma.array(pixel_median, mask=np.isnan(pixel_median))
pixel_std = np.ma.array(pixel_std, mask=np.isnan(pixel_std))

unused_values = np.abs(masked_trace_integral - pixel_mean) > (max_sigma * pixel_std)

# only warn for values discard in the sigma clipping, not those from before
outliers = unused_values & (~masked_trace_integral.mask)
check_outlier_mask(outliers, self.log, "pedestal")
Expand Down Expand Up @@ -301,18 +307,13 @@ def calculate_pedestal_results(self, trace_integral, masked_pixels_of_sample):
deviation < self.charge_median_cut_outliers[0] * std_of_pixel_median[:,np.newaxis],
deviation > self.charge_median_cut_outliers[1] * std_of_pixel_median[:,np.newaxis],
)

# mask pixels with NaN mean, due to missing statistics
pixels_without_stat = np.where(np.isnan(pixel_mean)==True)
charge_median_outliers[pixels_without_stat] = True
charge_std_outliers[pixels_without_stat] = True


return {
'charge_median': np.ma.getdata(pixel_median),
'charge_mean': np.ma.getdata(pixel_mean),
'charge_std': np.ma.getdata(pixel_std),
rlopezcoto marked this conversation as resolved.
Show resolved Hide resolved
'charge_std_outliers': np.ma.getdata(charge_std_outliers),
'charge_median_outliers': np.ma.getdata(charge_median_outliers)
'charge_std_outliers': charge_std_outliers.filled(True),
'charge_median_outliers': charge_median_outliers.filled(True)
}


Expand Down
Loading