diff --git a/pipelines/quickLook.yaml b/pipelines/quickLook.yaml index a48c0ccef..ba7be659f 100644 --- a/pipelines/quickLook.yaml +++ b/pipelines/quickLook.yaml @@ -4,3 +4,5 @@ tasks: class: lsst.summit.utils.quickLook.QuickLookIsrTask config: doRepairCosmics: true + doCorrectGains: false + doDeferredCharge: false diff --git a/python/lsst/summit/utils/bestEffort.py b/python/lsst/summit/utils/bestEffort.py index cfcf56f52..ad7d3b452 100644 --- a/python/lsst/summit/utils/bestEffort.py +++ b/python/lsst/summit/utils/bestEffort.py @@ -26,7 +26,7 @@ import lsst.afw.image as afwImage import lsst.daf.butler as dafButler from lsst.daf.butler.registry import ConflictingDefinitionError -from lsst.ip.isr import IsrTask +from lsst.ip.isr import IsrTaskLSST from lsst.pex.config import Config from lsst.summit.utils.butlerUtils import getLatissDefaultCollections from lsst.summit.utils.quickLook import QuickLookIsrTask @@ -58,7 +58,7 @@ class BestEffortIsr: Extra collections to add to the butler init. Collections are prepended. defaultExtraIsrOptions : `dict`, optional A dict of extra isr config options to apply. Each key should be an - attribute of an isrTaskConfigClass. + attribute of an isrTaskLSSTConfigClass. doRepairCosmics : `bool`, optional Repair cosmic ray hits? doWrite : `bool`, optional @@ -238,7 +238,7 @@ def getExposure( raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None # default options that are probably good for most engineering time - isrConfig = IsrTask.ConfigClass() + isrConfig = IsrTaskLSST.ConfigClass() with importlib.resources.path("lsst.summit.utils", "resources/config/quickLookIsr.py") as cfgPath: isrConfig.load(cfgPath) diff --git a/python/lsst/summit/utils/guiders/detection.py b/python/lsst/summit/utils/guiders/detection.py index 4f17b3c2b..a2c95bdc9 100644 --- a/python/lsst/summit/utils/guiders/detection.py +++ b/python/lsst/summit/utils/guiders/detection.py @@ -37,8 +37,9 @@ from astropy.nddata import Cutout2D from astropy.stats import sigma_clipped_stats +import lsst.afw.detection as afwDetect from lsst.afw.image import ExposureF, ImageF, MaskedImageF -from lsst.summit.utils.utils import detectObjectsInExp +from lsst.afw.math import STDEVCLIP, makeStatistics from .reading import GuiderData @@ -277,6 +278,7 @@ def runSourceDetection( cutOutSize: int = 25, apertureRadius: int = 5, gain: float = 1.0, + nPixMin: int = 10, ) -> pd.DataFrame: """ Detect sources in an image and measure their properties. @@ -293,6 +295,8 @@ def runSourceDetection( Aperture radius in pixels for photometry. gain : `float` Detector gain (e-/ADU). + nPixMin : `int` + Minimum number of pixels in a footprint for detection. Returns ------- @@ -302,13 +306,26 @@ def runSourceDetection( # Step 1: Convert numpy image to MaskedImage and Exposure exposure = ExposureF(MaskedImageF(ImageF(image))) - # Step 2: Detect sources - # we assume that we have bright stars - # filter out stamps with no stars + # Step 2: Detect sources using STDEVCLIP for the background noise. + # The input coadd images are dithered (see GuiderData.getStampArrayCoadd) + # to prevent integer quantization from collapsing the pixel distribution + # and causing STDEVCLIP to return 0. (See DM-54263.) + footprints = None if not isBlankImage(image): - footprints = detectObjectsInExp(exposure, nSigma=threshold) - else: - footprints = None + median = np.nanmedian(image) + exposure.image -= median + imageStd = float(makeStatistics(exposure.getMaskedImage(), STDEVCLIP).getValue(STDEVCLIP)) + if imageStd <= 0: + # Fallback: sigma68 is robust to quantization and bright stars. + p16, p84 = np.nanpercentile(image, [16, 84]) + imageStd = (p84 - p16) / 2.0 + if imageStd <= 0: + exposure.image += median + return pd.DataFrame(columns=DEFAULT_COLUMNS) + absThreshold = threshold * imageStd + thresh = afwDetect.Threshold(absThreshold, afwDetect.Threshold.VALUE) + footprints = afwDetect.FootprintSet(exposure.getMaskedImage(), thresh, "DETECTED", nPixMin) + exposure.image += median if not footprints: return pd.DataFrame(columns=DEFAULT_COLUMNS) @@ -613,7 +630,6 @@ def buildReferenceCatalog( apertureRadius = int(config.aperSizeArcsec / pixelScale) array = guiderData.getStampArrayCoadd(guiderName) - # array = np.where(array < 0, 0, array) # Ensure no negative values array = array - np.nanmin(array) # Ensure no negative values sources = runSourceDetection( array, diff --git a/python/lsst/summit/utils/guiders/reading.py b/python/lsst/summit/utils/guiders/reading.py index d671fb5f0..2ca652068 100644 --- a/python/lsst/summit/utils/guiders/reading.py +++ b/python/lsst/summit/utils/guiders/reading.py @@ -406,10 +406,15 @@ def getStampArrayCoadd(self, detName: str) -> np.ndarray: if len(stamps) == 0: raise ValueError(f"No stamps found for detector {detName!r}") - # Collect arrays, with optional bias subtraction + # Collect arrays, with optional bias subtraction. arrList = [self[detName, idx] for idx in range(len(stamps))] - stack = np.nanmedian(arrList, axis=0) - return stack + # Add a uniform [0, 1) dither per stamp before the median to break + # integer quantization. Without this, the coadd pixel distribution + # can be so narrow that STDEVCLIP returns 0. (See DM-54263.) + stack = np.array(arrList, dtype=np.float32) + rng = np.random.default_rng(seed=0) + stack += rng.uniform(0, 1, size=stack.shape).astype(np.float32) + return np.nanmedian(stack, axis=0) def getGuiderAmpName(self, detName: str) -> str: """ diff --git a/python/lsst/summit/utils/quickLook.py b/python/lsst/summit/utils/quickLook.py index c309961ef..6237ceb41 100644 --- a/python/lsst/summit/utils/quickLook.py +++ b/python/lsst/summit/utils/quickLook.py @@ -23,23 +23,21 @@ import importlib.resources from typing import Any -import numpy as np - import lsst.afw.cameraGeom as camGeom import lsst.afw.image as afwImage import lsst.ip.isr as ipIsr import lsst.pex.config as pexConfig import lsst.pipe.base as pipeBase import lsst.pipe.base.connectionTypes as cT -from lsst.ip.isr import IsrTask -from lsst.ip.isr.isrTask import IsrTaskConnections +from lsst.ip.isr import IsrTaskLSST +from lsst.ip.isr.isrTaskLSST import IsrTaskLSSTConnections from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask from lsst.pipe.tasks.characterizeImage import CharacterizeImageTask __all__ = ["QuickLookIsrTask", "QuickLookIsrTaskConfig"] -class QuickLookIsrTaskConnections(IsrTaskConnections): +class QuickLookIsrTaskConnections(IsrTaskLSSTConnections): """Copy isrTask's connections, changing prereq min values to zero. Copy all the connections directly for IsrTask, keeping ccdExposure as @@ -50,7 +48,9 @@ class QuickLookIsrTaskConnections(IsrTaskConnections): def __init__(self, *, config: Any = None): # programatically clone all of the connections from isrTask # setting minimum values to zero for everything except the ccdExposure - super().__init__(config=IsrTask.ConfigClass()) # need a dummy config, isn't used other than for ctor + super().__init__( + config=IsrTaskLSST.ConfigClass() + ) # need a dummy config, isn't used other than for ctor for name, connection in self.allConnections.items(): if hasattr(connection, "minimum"): setattr( @@ -95,13 +95,16 @@ class QuickLookIsrTask(pipeBase.PipelineTask): config: QuickLookIsrTaskConfig _DefaultName = "quickLook" - def __init__(self, isrTask: IsrTask = IsrTask, **kwargs: Any): + def __init__(self, isrTask: IsrTaskLSST = IsrTaskLSST, **kwargs: Any): super().__init__(**kwargs) # Pass in IsrTask so that we can modify it slightly for unit tests. # Note that this is not an instance of the IsrTask class, but the class # itself, which is then instantiated later on, in the run() method, # with the dynamically generated config. - self.isrTask = IsrTask + # import pdb; pdb.set_trace() + if IsrTaskLSST._DefaultName != "isrLSST": + raise RuntimeError("QuickLookIsrTask should now always use IsrTaskLSST for processing.") + self.isrTask = IsrTaskLSST def run( self, @@ -111,22 +114,14 @@ def run( bias: afwImage.Exposure | None = None, dark: afwImage.Exposure | None = None, flat: afwImage.Exposure | None = None, - fringes: afwImage.Exposure | None = None, defects: ipIsr.Defects | None = None, linearizer: ipIsr.linearize.LinearizeBase | None = None, crosstalk: ipIsr.crosstalk.CrosstalkCalib | None = None, - bfKernel: np.ndarray | None = None, - newBFKernel: ipIsr.BrighterFatterKernel | None = None, + bfKernel: ipIsr.BrighterFatterKernel | None = None, ptc: ipIsr.PhotonTransferCurveDataset | None = None, - crosstalkSources: list | None = None, - isrBaseConfig: ipIsr.IsrTaskConfig | None = None, - filterTransmission: afwImage.TransmissionCurve | None = None, - opticsTransmission: afwImage.TransmissionCurve | None = None, - strayLightData: Any | None = None, - sensorTransmission: afwImage.TransmissionCurve | None = None, - atmosphereTransmission: afwImage.TransmissionCurve | None = None, + isrBaseConfig: ipIsr.IsrTaskLSSTConfig | None = None, deferredChargeCalib: Any | None = None, - illumMaskedImage: afwImage.MaskedImage | None = None, + gainCorrection: ipIsr.IsrCalib | None = None, ) -> pipeBase.Struct: """Run isr and cosmic ray repair using, doing as much isr as possible. @@ -160,16 +155,14 @@ def run( Functor for linearization. crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional Calibration for crosstalk. - bfKernel : `numpy.ndarray`, optional - Brighter-fatter kernel. - newBFKernel : `ipIsr.BrighterFatterKernel`, optional + bfKernel : `ipIsr.BrighterFatterKernel`, optional New Brighter-fatter kernel. ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional Photon transfer curve dataset, with, e.g., gains and read noise. - crosstalkSources : `list`, optional - List of possible crosstalk sources. - isrBaseConfig : `lsst.ip.isr.IsrTaskConfig`, optional + cti : `lsst.ip.isr.DeferredChargeCalib`, optional + Charge transfer inefficiency correction calibration. + isrBaseConfig : `lsst.ip.isr.IsrTaskLSSTConfig`, optional An isrTask config to act as the base configuration. Options which involve applying a calibration product are ignored, but this allows for the configuration of e.g. the number of overscan columns. @@ -195,6 +188,9 @@ def run( Gains used to override the detector's nominal gains for the brighter-fatter correction. A dict keyed by amplifier name for the detector in question. + gainCorrection : `lsst.ip.isr.IsrCalib`, optional + Gain correction calibration, to account for small + time-dependent shifts. Returns ------- @@ -204,7 +200,7 @@ def run( The ISRed and cosmic-ray-repaired exposure. """ if not isrBaseConfig: - isrConfig = IsrTask.ConfigClass() + isrConfig = IsrTaskLSST.ConfigClass() with importlib.resources.path("lsst.summit.utils", "resources/config/quickLookIsr.py") as cfgPath: isrConfig.load(cfgPath) else: @@ -213,12 +209,12 @@ def run( isrConfig.doBias = False isrConfig.doDark = False isrConfig.doFlat = False - isrConfig.doFringe = False isrConfig.doDefect = False isrConfig.doLinearize = False isrConfig.doCrosstalk = False isrConfig.doBrighterFatter = False - isrConfig.usePtcGains = False + isrConfig.doDeferredCharge = False + isrConfig.doCorrectGains = False if bias: isrConfig.doBias = True @@ -232,10 +228,6 @@ def run( isrConfig.doFlat = True self.log.info("Running with flat correction") - if fringes: - isrConfig.doFringe = True - self.log.info("Running with fringe correction") - if defects: isrConfig.doDefect = True self.log.info("Running with defect correction") @@ -248,31 +240,32 @@ def run( isrConfig.doCrosstalk = True self.log.info("Running with crosstalk correction") - if newBFKernel is not None: - bfGains = newBFKernel.gain - isrConfig.doBrighterFatter = True - self.log.info("Running with new brighter-fatter correction") - else: - bfGains = None - - if bfKernel is not None and bfGains is None: + if bfKernel is not None: isrConfig.doBrighterFatter = True self.log.info("Running with brighter-fatter correction") - if ptc: - isrConfig.usePtcGains = True - self.log.info("Running with ptc correction") + if deferredChargeCalib is not None: + isrConfig.doDeferredCharge = True + self.log.info("Running with CTI correction") + + if gainCorrection is not None: + isrConfig.doCorrectGains = True + self.log.info("Running with Gain corrections") + + if ptc is None: + raise RuntimeError("IsrTaskLSST requires a PTC.") - isrConfig.doWrite = False isrTask = self.isrTask(config=isrConfig) - if fringes: - # Must be run after isrTask is instantiated. - isrTask.fringe.loadFringes( - fringes, - expId=ccdExposure.info.id, - assembler=isrTask.assembleCcd if isrConfig.doAssembleIsrExposures else None, - ) + # DM-47959: TODO Add fringe correction to IsrTaskLSST. + # if fringes: + # # Must be run after isrTask is instantiated. + # isrTask.fringe.loadFringes( + # fringes, + # expId=ccdExposure.info.id, + # assembler=isrTask.assembleCcd if + # isrConfig.doAssembleIsrExposures else None, + # ) result = isrTask.run( ccdExposure, @@ -280,21 +273,13 @@ def run( bias=bias, dark=dark, flat=flat, - fringes=fringes, defects=defects, linearizer=linearizer, crosstalk=crosstalk, bfKernel=bfKernel, - bfGains=bfGains, ptc=ptc, - crosstalkSources=crosstalkSources, - filterTransmission=filterTransmission, - opticsTransmission=opticsTransmission, - sensorTransmission=sensorTransmission, - atmosphereTransmission=atmosphereTransmission, - strayLightData=strayLightData, deferredChargeCalib=deferredChargeCalib, - illumMaskedImage=illumMaskedImage, + gainCorrection=gainCorrection, ) postIsr = result.exposure diff --git a/python/lsst/summit/utils/resources/config/quickLookIsr.py b/python/lsst/summit/utils/resources/config/quickLookIsr.py index f08a75d10..af7ba9698 100644 --- a/python/lsst/summit/utils/resources/config/quickLookIsr.py +++ b/python/lsst/summit/utils/resources/config/quickLookIsr.py @@ -1,8 +1,9 @@ # mypy: disable-error-code="name-defined" -config.doWrite = False # this task writes separately, no need for this config.doSaturation = True # saturation very important for roundness measurement in qfm -config.doSaturationInterpolation = True -config.overscan.fitType = "MEDIAN_PER_ROW" -config.overscan.doParallelOverscan = True config.brighterFatterMaxIter = 2 # Uncomment this to remove test warning +config.doDeferredCharge = False # no calib for this yet +config.doBootstrap = False +config.doApplyGains = True +config.doSuspect = False +config.defaultSaturationSource = "CAMERAMODEL" diff --git a/python/lsst/summit/utils/utils.py b/python/lsst/summit/utils/utils.py index 471ef6eaa..ec8750e16 100644 --- a/python/lsst/summit/utils/utils.py +++ b/python/lsst/summit/utils/utils.py @@ -488,6 +488,7 @@ def _getAltAzZenithsFromSeqNum( for seqNum in seqNumList: md = butler.get("raw.metadata", day_obs=dayObs, seq_num=seqNum, detector=0) obsInfo = ObservationInfo(md) + assert obsInfo.altaz_begin is not None, f"altaz_begin is None for seqNum={seqNum}" alt = obsInfo.altaz_begin.alt.value az = obsInfo.altaz_begin.az.value elevations.append(alt) diff --git a/tests/test_bestEffortIsr.py b/tests/test_bestEffortIsr.py index 82f38de8b..6923c67b8 100644 --- a/tests/test_bestEffortIsr.py +++ b/tests/test_bestEffortIsr.py @@ -40,7 +40,7 @@ def setUpClass(cls): # NCSA - LATISS/raw/all # TTS - LATISS-test-data-tts # summit - LATISS_test_data - cls.dataId = {"day_obs": 20210121, "seq_num": 743, "detector": 0} + cls.dataId = {"day_obs": 20250902, "seq_num": 10, "detector": 0} def test_getExposure(self): # in most locations this will load a pre-made image diff --git a/tests/test_quickLook.py b/tests/test_quickLook.py index d2fe2a80a..fdd82e700 100644 --- a/tests/test_quickLook.py +++ b/tests/test_quickLook.py @@ -23,17 +23,14 @@ import tempfile import unittest -import numpy as np - import lsst.afw.cameraGeom.testUtils as afwTestUtils import lsst.afw.image as afwImage import lsst.daf.butler.tests as butlerTests import lsst.ip.isr as ipIsr -import lsst.ip.isr.isrMock as isrMock +import lsst.ip.isr.isrMockLSST as isrMock import lsst.pex.exceptions import lsst.pipe.base as pipeBase import lsst.pipe.base.testUtils -from lsst.afw.image import TransmissionCurve from lsst.summit.utils.quickLook import QuickLookIsrTask, QuickLookIsrTaskConfig @@ -41,25 +38,23 @@ class QuickLookIsrTaskTestCase(unittest.TestCase): """Tests of the run method with fake data.""" def setUp(self): - self.mockConfig = isrMock.IsrMockConfig() - self.dataContainer = isrMock.MockDataContainer(config=self.mockConfig) - self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera() + self.mockConfig = isrMock.IsrMockLSSTConfig() + self.camera = isrMock.IsrMockLSST(config=self.mockConfig).getCamera() + self.ccdExposure = isrMock.RawMockLSST(config=self.mockConfig).run() - self.ccdExposure = isrMock.RawMock(config=self.mockConfig).run() self.detector = self.ccdExposure.getDetector() amps = self.detector.getAmplifiers() ampNames = [amp.getName() for amp in amps] # # Mock other optional parameters - self.bias = self.dataContainer.get("bias") - self.dark = self.dataContainer.get("dark") - self.flat = self.dataContainer.get("flat") - self.defects = self.dataContainer.get("defects") + self.bias = isrMock.BiasMockLSST(config=self.mockConfig).run() + self.dark = isrMock.DarkMockLSST(config=self.mockConfig).run() + self.flat = isrMock.FlatMockLSST(config=self.mockConfig).run() + self.defects = isrMock.DefectMockLSST(config=self.mockConfig).run() self.ptc = ipIsr.PhotonTransferCurveDataset(ampNames=ampNames) # Mock PTC dataset - self.bfKernel = self.dataContainer.get("bfKernel") - self.newBFKernel = pipeBase.Struct(gain={}) - for amp_i, amp in enumerate(ampNames): - self.newBFKernel.gain[amp] = 0.9 + 0.1 * amp_i + for amp, gain in self.mockConfig.gainDict.items(): + self.ptc.gain[amp] = 1.0 + self.bfKernel = isrMock.BfKernelMockLSST(config=self.mockConfig).run() self.task = QuickLookIsrTask(config=QuickLookIsrTaskConfig()) def test_runQuickLook(self): @@ -74,9 +69,7 @@ def test_runQuickLook(self): linearizer=None, crosstalk=None, bfKernel=self.bfKernel, - newBFKernel=self.newBFKernel, ptc=self.ptc, - crosstalkSources=None, ) self.assertIsNotNone(result, "Result of run method should not be None") self.assertIsInstance(result, pipeBase.Struct, "Result should be of type lsst.pipe.base.Struct") @@ -88,7 +81,9 @@ def test_runQuickLook(self): def test_runQuickLookMissingData(self): # Test without any inputs other than the exposure - result = self.task.run(self.ccdExposure) + # And the PTC. That's required now. + + result = self.task.run(self.ccdExposure, ptc=self.ptc) self.assertIsInstance(result.exposure, afwImage.Exposure) def test_runQuickLookBadDark(self): @@ -129,23 +124,16 @@ def setUp(self): dark = "dark" flat = "flat" defects = "defects" - bfKernel = "bfKernel" - newBFKernel = "brighterFatterKernel" + bfKernel = "bfk" ptc = "ptc" - filterTransmission = "transmission_filter" - deferredChargeCalib = "cpCtiCalib" - opticsTransmission = "transmission_optics" - strayLightData = "yBackground" - atmosphereTransmission = "transmission_atmosphere" + deferredChargeCalib = "cti" crosstalk = "crosstalk" - illumMaskedImage = "illum" linearizer = "linearizer" - fringes = "fringe" - sensorTransmission = "transmission_sensor" - crosstalkSources = "isrOverscanCorrected" + gainCorrection = "gain_correction" # outputs outputExposure = "postISRCCD" + outputStatistics = "isrStatistics" # quickLook-only outputs exposure = "quickLookExp" @@ -170,37 +158,19 @@ def setUp(self): butlerTests.addDatasetType(self.repo, defects, {"instrument", "detector"}, "Defects") butlerTests.addDatasetType(self.repo, linearizer, {"instrument", "detector"}, "Linearizer") butlerTests.addDatasetType(self.repo, crosstalk, {"instrument", "detector"}, "CrosstalkCalib") - butlerTests.addDatasetType(self.repo, bfKernel, {"instrument"}, "NumpyArray") - butlerTests.addDatasetType(self.repo, newBFKernel, {"instrument", "detector"}, "BrighterFatterKernel") + butlerTests.addDatasetType(self.repo, bfKernel, {"instrument", "detector"}, "BrighterFatterKernel") butlerTests.addDatasetType(self.repo, ptc, {"instrument", "detector"}, "PhotonTransferCurveDataset") - butlerTests.addDatasetType( - self.repo, filterTransmission, {"instrument", "physical_filter"}, "TransmissionCurve" - ) - butlerTests.addDatasetType(self.repo, opticsTransmission, {"instrument"}, "TransmissionCurve") butlerTests.addDatasetType(self.repo, deferredChargeCalib, {"instrument", "detector"}, "IsrCalib") - butlerTests.addDatasetType( - self.repo, strayLightData, {"instrument", "physical_filter", "detector"}, "Exposure" - ) - butlerTests.addDatasetType(self.repo, atmosphereTransmission, {"instrument"}, "TransmissionCurve") - butlerTests.addDatasetType( - self.repo, illumMaskedImage, {"instrument", "physical_filter", "detector"}, "MaskedImage" - ) - butlerTests.addDatasetType( - self.repo, fringes, {"instrument", "physical_filter", "detector"}, "Exposure" - ) - butlerTests.addDatasetType( - self.repo, sensorTransmission, {"instrument", "detector"}, "TransmissionCurve" - ) - butlerTests.addDatasetType( - self.repo, crosstalkSources, {"instrument", "exposure", "detector"}, "Exposure" - ) + butlerTests.addDatasetType(self.repo, gainCorrection, {"instrument", "detector"}, "IsrCalib") # outputs butlerTests.addDatasetType( self.repo, outputExposure, {"instrument", "exposure", "detector"}, "Exposure" ) butlerTests.addDatasetType(self.repo, exposure, {"instrument", "exposure", "detector"}, "Exposure") - + butlerTests.addDatasetType( + self.repo, outputStatistics, {"instrument", "exposure", "detector"}, "StructuredDataDict" + ) # dataIds self.exposure_id = self.repo.registry.expandDataId( { @@ -220,38 +190,57 @@ def setUp(self): ) # put empty data - transmissionCurve = TransmissionCurve.makeSpatiallyConstant( - np.ones(2), np.linspace(0, 1, 2), 0.0, 0.0 - ) self.butler = butlerTests.makeTestCollection(self.repo) - self.butler.put(afwImage.ExposureF(), ccdExposure, self.exposure_id) + + myExposure = isrMock.RawMockLSST().run() + self.butler.put(myExposure, ccdExposure, self.exposure_id) + self.butler.put(afwTestUtils.CameraWrapper().camera, camera, self.instrument_id) - self.butler.put(afwImage.ExposureF(), bias, self.detector_id) - self.butler.put(afwImage.ExposureF(), dark, self.detector_id) - self.butler.put(afwImage.ExposureF(), flat, self.flat_id) + + myBias = isrMock.BiasMockLSST().run() + self.butler.put(myBias, bias, self.detector_id) + + myDark = isrMock.DarkMockLSST().run() + self.butler.put(myDark, dark, self.detector_id) + + myFlat = isrMock.FlatMockLSST().run() + self.butler.put(myFlat, flat, self.flat_id) + self.butler.put(lsst.ip.isr.Defects(), defects, self.detector_id) - self.butler.put(np.zeros(2), bfKernel, self.instrument_id) - self.butler.put( - lsst.ip.isr.brighterFatterKernel.BrighterFatterKernel(), newBFKernel, self.detector_id - ) - self.butler.put(ipIsr.PhotonTransferCurveDataset(), ptc, self.detector_id) - self.butler.put(transmissionCurve, filterTransmission, self.filter_id) - self.butler.put(lsst.ip.isr.calibType.IsrCalib(), deferredChargeCalib, self.detector_id) - self.butler.put(transmissionCurve, opticsTransmission, self.instrument_id) - self.butler.put(afwImage.ExposureF(), strayLightData, self.flat_id) - self.butler.put(transmissionCurve, atmosphereTransmission, self.instrument_id) - self.butler.put(lsst.ip.isr.crosstalk.CrosstalkCalib(), crosstalk, self.detector_id) - self.butler.put(afwImage.ExposureF().maskedImage, illumMaskedImage, self.flat_id) + + myBfk = isrMock.BfKernelMockLSST().run() + self.butler.put(myBfk, "bfk", self.detector_id) + + myPtc = ipIsr.PhotonTransferCurveDataset() + for amp in myExposure.getDetector(): + myPtc.gain[amp.getName()] = myBfk.gain[amp.getName()] + myPtc.noise[amp.getName()] = 5.0 + myPtc.ptcTurnoff[amp.getName()] = 100_000 + self.butler.put(myPtc, ptc, self.detector_id) + + myCti = lsst.ip.isr.deferredCharge.DeferredChargeCalib() + for amp in myExposure.getDetector(): + myCti.driftScale[amp.getName()] = 1.8e-5 + myCti.decayTime[amp.getName()] = 3.0 + myCti.globalCti[amp.getName()] = 1.5e-7 + myCti.serialTraps[amp.getName()] = lsst.ip.isr.deferredCharge.SerialTrap( + 10, 3, 200, "linear", [5, 1e-3] + ) + self.butler.put(myCti, "cti", self.detector_id) + self.butler.put(lsst.ip.isr.gainCorrection.GainCorrection(), gainCorrection, self.detector_id) + + nAmp = len(myExposure.getDetector()) + myCrosstalk = lsst.ip.isr.crosstalk.CrosstalkCalib(nAmp=nAmp) + myCrosstalk.hasCrosstalk = True + self.butler.put(myCrosstalk, crosstalk, self.detector_id) + self.butler.put(lsst.ip.isr.linearize.Linearizer(), linearizer, self.detector_id) - self.butler.put(afwImage.ExposureF(), fringes, self.flat_id) - self.butler.put(transmissionCurve, sensorTransmission, self.detector_id) - self.butler.put(afwImage.ExposureF(), crosstalkSources, self.exposure_id) def tearDown(self): del self.repo_path # this removes the temporary directory def test_runQuantum(self): - config = ipIsr.IsrTaskConfig() + config = ipIsr.IsrTaskLSSTConfig() # Remove some outputs config.doBinnedExposures = False config.doSaveInterpPixels = False @@ -260,19 +249,18 @@ def test_runQuantum(self): config.doCalculateStatistics = False # Turn on all optional inputs - config.doAttachTransmissionCurve = True - config.doIlluminationCorrection = True - config.doStrayLight = True - config.doDeferredCharge = True - config.usePtcReadNoise = True + config.doDeferredCharge = False # There is no CTI calibration available for LATISS. config.doCrosstalk = True config.doBrighterFatter = True + # Disable things that do not work with dummy inputs. + config.doCorrectGains = False + # Override a method in IsrTask that is executed early, to instead raise # a custom exception called ExitMock that we can catch and ignore. - isrTask = ipIsr.IsrTask + isrTask = ipIsr.IsrTaskLSST isrTask.ensureExposure = raiseExitMockError - task = QuickLookIsrTask(isrTask=isrTask) + task = QuickLookIsrTask() lsst.pipe.base.testUtils.assertValidInitOutput(task) # Use the names of the connections here, not the Butler dataset name @@ -287,25 +275,19 @@ def test_runQuantum(self): "dark": self.detector_id, "flat": self.flat_id, "defects": self.detector_id, - "bfKernel": self.instrument_id, - "newBFKernel": self.detector_id, + "bfKernel": self.detector_id, "ptc": self.detector_id, - "filterTransmission": self.filter_id, "deferredChargeCalib": self.detector_id, - "opticsTransmission": self.instrument_id, - "strayLightData": self.flat_id, - "atmosphereTransmission": self.instrument_id, "crosstalk": self.detector_id, - "illumMaskedImage": self.flat_id, "linearizer": self.detector_id, "fringes": self.flat_id, - "sensorTransmission": self.detector_id, - "crosstalkSources": [self.exposure_id, self.exposure_id], + "gainCorrection": self.detector_id, # outputs "outputExposure": self.exposure_id, - "exposure": self.exposure_id, + "outputStatistics": self.exposure_id, }, ) + # Check that the proper kwargs are passed to run(). with contextlib.suppress(ExitMockError): lsst.pipe.base.testUtils.runTestQuantum(task, self.butler, quantum, mockRun=False) @@ -320,3 +302,16 @@ class ExitMockError(Exception): """A custom exception to catch during a unit test.""" pass + + +class TestMemory(lsst.utils.tests.MemoryTestCase): + pass + + +def setup_module(module): + lsst.utils.tests.init() + + +if __name__ == "__main__": + lsst.utils.tests.init() + unittest.main()