Pipeline / pipeline / 18dd01b3a70

Commits

Vincent Geers authored 18dd01b3a70 12 Dec 2024

PIPE-2394: update docstrings, type hints, commentary.

pipeline/h/tasks/tsyscal/tsyscal.py

Modified

-1
+import collections
-2
+from operator import itemgetter, attrgetter
--
+from typing import Dict, List, Optional, Set
-3
-4
+import pipeline.infrastructure as infrastructure
-5
+import pipeline.infrastructure.basetask as basetask
-6
+import pipeline.infrastructure.callibrary as callibrary
-7
+import pipeline.infrastructure.utils as utils
-8
+import pipeline.infrastructure.vdp as vdp
-9
+from pipeline.domain import MeasurementSet
-10
+from pipeline.h.heuristics import caltable as caltable_heuristic
-11
+from pipeline.h.heuristics.tsysspwmap import tsysspwmap
-12
+from pipeline.infrastructure import casa_tasks
-83
+            'vis': self.vis,
-84
+            'caltable': self.caltable
-85
+        }
-86
-87
-88
+@task_registry.set_equivalent_casa_task('h_tsyscal')
-89
+@task_registry.set_casa_commands_comment('The Tsys calibration and spectral window map is computed.')
-90
+class Tsyscal(basetask.StandardTaskTemplate):
-91
+    Inputs = TsyscalInputs
-92
--
+    def prepare(self):
-+
+    def prepare(self) -> resultobjects.TsyscalResults:
-94
+        inputs = self.inputs
-95
-96
+        # make a note of the current inputs state before we start fiddling
-97
+        # with it. This origin will be attached to the final CalApplication.
-98
+        origin = callibrary.CalAppOrigin(task=Tsyscal, inputs=inputs.to_casa_args())
-99
-100
+        # construct the Tsys cal file
-101
+        gencal_args = inputs.to_casa_args()
-102
+        gencal_job = casa_tasks.gencal(caltype='tsys', **gencal_args)
-103
+        self._executor.execute(gencal_job)
-108
+        LOG.todo('tsysspwmap heuristic won\'t handle missing file')
-109
+        nospwmap, spwmap = tsysspwmap(ms=inputs.ms, tsystable=tsys_table, tsysChanTol=inputs.chantol)
-110
-111
+        calfrom_defaults = dict(caltype='tsys', spwmap=spwmap, interp='linear,linear')
-112
-113
+        is_single_dish = utils.contains_single_dish(inputs.context)
-114
+        calapps = get_calapplications(inputs.ms, tsys_table, calfrom_defaults, origin, spwmap, is_single_dish)
-115
-116
+        return resultobjects.TsyscalResults(pool=calapps, unmappedspws=nospwmap)
-117
--
+    def analyse(self, result):
-+
+    def analyse(self, result: resultobjects.TsyscalResults) -> resultobjects.TsyscalResults:
-119
+        # double-check that the caltable was actually generated
-120
+        on_disk = [ca for ca in result.pool if ca.exists()]
-121
+        result.final[:] = on_disk
-122
-123
+        missing = [ca for ca in result.pool if ca not in on_disk]
-124
+        result.error.clear()
-125
+        result.error.update(missing)
-126
-127
+        return result
-128
-129
-130
+# Holds an observing intent and the preferred/fallback gainfield args to be used for that intent
-131
+GainfieldMapping = collections.namedtuple('GainfieldMapping', 'intent preferred fallback')
-132
-133
--
+def get_solution_map(ms: MeasurementSet, is_single_dish: bool) -> List[GainfieldMapping]:
-+
+def get_solution_map(ms: MeasurementSet, is_single_dish: bool) -> list[GainfieldMapping]:
-135
+    """
-136
+    Get gainfield solution map. Different solution maps are returned for
-137
+    single dish and interferometric data.
-138
--
+    :param ms: MS to analyse
--
+    :param is_single_dish: True if MS is single dish data
--
+    :return: list of GainfieldMappings
-+
+    Args:
-+
+        ms: MS to analyse.
-+
+        is_single_dish: True if MS is single dish data.
-+
-+
+    Returns:
-+
+        List of GainfieldMappings.
-145
+    """
-146
+    # define function to get Tsys fields for intent
--
+    def f(intent, exclude: Optional[str] = None):
-+
+    def f(intent, exclude: str | None = None) -> str:
-148
+        if ',' in intent:
-149
+            head, tail = intent.split(',', 1)
-150
+            # the 'if o' test filters out results for intents that do not have
-151
+            # fields, e.g., PHASE for SD data
-152
+            return ','.join(o for o in (f(head, exclude=exclude), f(tail, exclude=exclude)) if o)
--
+        return ','.join(str(s) for s in get_tsys_fields_for_intent(ms, intent, exclude=exclude))
-+
+        return ','.join(str(s) for s in get_tsys_fields_for_intent(ms, intent, exclude_intents=exclude))
-154
-155
+    # return different gainfield maps for single dish and interferometric
-156
+    if is_single_dish:
-157
+        return [
-158
+            GainfieldMapping(intent='BANDPASS', preferred=f('BANDPASS'), fallback='nearest'),
-159
+            GainfieldMapping(intent='AMPLITUDE', preferred=f('AMPLITUDE'), fallback='nearest'),
-160
+            # non-empty magic string to differentiate between no field found and a null fallback
-161
+            GainfieldMapping(intent='TARGET', preferred=f('TARGET'), fallback='___EMPTY_STRING___')
-162
+        ]
-163
-164
+    else:
--
+        # Intent mapping extracted from CAS-12213 ticket.
-+
+        # CAS-12213: original intent mapping.
-166
+        # PIPE-2080: updated to add mapping for DIFFGAINREF, DIFFGAINSRC intent.
-+
+        # PIPE-2394: updated mapping for PHASE, TARGET, CHECK
-168
+        #
--
+        # ObjectToBeCalibrated 	TsysSolutionToUse 	IfNoSolutionPresentThenUse
--
+        # BANDPASS cal 	        all BANDPASS cals 	fallback to 'nearest'
--
+        # FLUX cal 	            all FLUX cals 	    fallback to 'nearest'
--
+        # DIFFGAIN[REF|SRC]     all DIFFGAIN cals 	fallback to BANDPASS
--
+        # PHASE cal 	        all PHASE cals 	    all TARGETs
--
+        # TARGET 	            all TARGETs 	    all PHASE cals
--
+        # CHECK_SOURCE        	all TARGETs     	all PHASE cals
-+
+        # Intent to be calibrated:
-+
+        # - BANDPASS cal
-+
+        #   * Preferred: all BANDPASS cals.
-+
+        #   * Fallback: 'nearest'.
-+
+        # - FLUX cal
-+
+        #   * Preferred: all FLUX cals.
-+
+        #   * Fallback: 'nearest'.
-+
+        # - DIFFGAIN[REF|SRC]
-+
+        #   * Preferred: all DIFFGAIN cals.
-+
+        #   * Fallback to BANDPASS.
-+
+        # - PHASE cal
-+
+        #   * Preferred: ATMOSPHERE cals, but excluding AMP, BP, DIFFGAIN*, POL* cals, and TARGET.
-+
+        #   * Fallback: ATMOSPHERE cals, but excluding AMP, BP, DIFFGAIN*, POL* cals.
-+
+        # - TARGET
-+
+        #   * Preferred: ATMOSPHERE cals, but excluding AMP, BP, DIFFGAIN*, POL* cals, and PHASE.
-+
+        #   * Fallback: ATMOSPHERE cals, but excluding AMP, BP, DIFFGAIN*, POL* cals.
-+
+        # - CHECK_SOURCE
-+
+        #   * Preferred: ATMOSPHERE cals, but excluding AMP, BP, DIFFGAIN*, POL* cals, and PHASE.
-+
+        #   * Fallback: ATMOSPHERE cals, but excluding AMP, BP, DIFFGAIN*, POL* cals.
-+
-+
+        # PIPE-2394: typical calibrator intents to avoid (all but PHASE)
-+
+        # matching searching for nearby Tsys field for PHASE, TARGET, and/or
-+
+        # CHECK.
-+
+        exclude_intents = 'AMPLITUDE,BANDPASS,DIFFGAINREF,DIFFGAINSRC,POLARIZATION,POLANGLE,POLLEAKAGE'
-+
-194
+        return [
-195
+            GainfieldMapping(intent='BANDPASS', preferred=f('BANDPASS'), fallback='nearest'),
-196
+            GainfieldMapping(intent='AMPLITUDE', preferred=f('AMPLITUDE'), fallback='nearest'),
-197
+            GainfieldMapping(intent='DIFFGAINREF', preferred=f('DIFFGAINREF'), fallback=f('BANDPASS')),
-198
+            GainfieldMapping(intent='DIFFGAINSRC', preferred=f('DIFFGAINSRC'), fallback=f('BANDPASS')),
--
+            GainfieldMapping(intent='PHASE',
--
+                             preferred=f('ATMOSPHERE', exclude='AMPLITUDE,BANDPASS,DIFFGAINREF,DIFFGAINSRC,POLARIZATION,POLANGLE,POLLEAKAGE,TARGET'),
--
+                             fallback=f('ATMOSPHERE', exclude='AMPLITUDE,BANDPASS,DIFFGAINREF,DIFFGAINSRC,POLARIZATION,POLANGLE,POLLEAKAGE')),
--
+            GainfieldMapping(intent='TARGET',
--
+                             preferred=f('ATMOSPHERE', exclude='AMPLITUDE,BANDPASS,DIFFGAINREF,DIFFGAINSRC,POLARIZATION,POLANGLE,POLLEAKAGE,PHASE'),
--
+                             fallback=f('ATMOSPHERE', exclude='AMPLITUDE,BANDPASS,DIFFGAINREF,DIFFGAINSRC,POLARIZATION,POLANGLE,POLLEAKAGE')),
--
+            GainfieldMapping(intent='CHECK',
--
+                             preferred=f('ATMOSPHERE', exclude='AMPLITUDE,BANDPASS,DIFFGAINREF,DIFFGAINSRC,POLARIZATION,POLANGLE,POLLEAKAGE,PHASE'),
--
+                             fallback=f('ATMOSPHERE', exclude='AMPLITUDE,BANDPASS,DIFFGAINREF,DIFFGAINSRC,POLARIZATION,POLANGLE,POLLEAKAGE')),
-+
+            GainfieldMapping(intent='PHASE', preferred=f('ATMOSPHERE', exclude=f'{exclude_intents},TARGET'),
-+
+                             fallback=f('ATMOSPHERE', exclude=exclude_intents)),
-+
+            GainfieldMapping(intent='TARGET', preferred=f('ATMOSPHERE', exclude=f'{exclude_intents},PHASE'),
-+
+                             fallback=f('ATMOSPHERE', exclude=exclude_intents)),
-+
+            GainfieldMapping(intent='CHECK', preferred=f('ATMOSPHERE', exclude=f'{exclude_intents},PHASE'),
-+
+                             fallback=f('ATMOSPHERE', exclude=exclude_intents)),
-205
+        ]
-206
-207
--
+def get_gainfield_map(ms: MeasurementSet, is_single_dish: bool) -> Dict:
-+
+def get_gainfield_map(ms: MeasurementSet, is_single_dish: bool) -> dict:
-209
+    """
-210
+    Get the mapping of observing intent to gainfield parameter for a
-211
+    measurement set.
-212
-213
+    The mapping follows the observing intent to gainfield intent defined in
-214
+    CAS-12213.
-215
--
+    :param ms: MS to analyse
--
+    :param is_single_dish: boolean for if SD data or not
--
+    :return: dict of {observing intent: gainfield}
--
+    """
-+
+    Args:
-+
+        ms: MS to analyse.
-+
+        is_single_dish: boolean for if SD data or not.
-219
-+
+    Returns:
-+
+        Dictionary of {observing intent: gainfield}.
-+
+    """
-223
+    soln_map = get_solution_map(ms, is_single_dish)
-224
+    final_map = {s.intent: s.preferred if s.preferred else s.fallback for s in soln_map}
-225
-226
+    # Detect cases where there's no preferred or fallback gainfield mapping,
-227
+    # e.g., if there are no Tsys scans on a target or phase calibrator.
-228
+    undefined_intents = [k for k, v in final_map.items()
-229
+                         if not v  # gainfield mapping is empty..
-230
+                         and k in ms.intents]  # ..for a valid intent in the MS
-231
+    if undefined_intents:
-232
+        msg = 'Undefined Tsys gainfield mapping for {} intents: {}'.format(ms.basename, undefined_intents)
-233
+        LOG.error(msg)
-234
+        raise AssertionError(msg)
-235
-236
+    # convert magic string back to empty string
-237
+    converted = {k: v.replace('___EMPTY_STRING___', '') for k, v in final_map.items()}
-238
-239
+    return converted
-240
-241
--
+def get_tsys_fields_for_intent(ms: MeasurementSet, intent: str, exclude: Optional[str] = None) -> Set[str]:
-+
+def get_tsys_fields_for_intent(ms: MeasurementSet, intent: str, exclude_intents: str | None = None) -> set[str]:
-243
+    """
-244
+    Returns the identity of the Tsys field(s) for an intent.
-245
--
+    :param ms: MS to analyse
--
+    :param intent: intent to retrieve fields for.
--
+    :return: set of field identifiers corresponding to intent
-+
+    Args:
-+
+        ms: MS to analyse.
-+
+        intent: Intent to retrieve fields for.
-+
+        exclude_intents: String of intent(s) (comma-separated) that should not
-+
+            be covered by the Tsys field.
-+
-+
+    Returns:
-+
+        Set of field identifiers corresponding to given intent, while not
-+
+        associated with intents (optionally) given by ``exclude_intents``.
-255
+    """
-256
+    # In addition to the science intent scan, a field must also have a Tsys
-257
+    # scan observed for a Tsys solution to be considered present. The
-258
+    # exception is science mosaics, which are handled as a special case.
-259
-260
+    # We need to know which science intent scans have Tsys scans; the ones
-261
+    # that don't will be checked for science mosaics separately. This lets
-262
+    # us handle single field, single pointing science targets alongside mosaic
-263
+    # targets mixed together in the same EB. Theoretically, at least...
-264
+    intent_fields = ms.get_fields(intent=intent)
--
+    if exclude is not None:
--
+        intent_fields = [f for f in intent_fields if f.intents.isdisjoint(set(exclude.split(',')))]
-+
-+
+    # PIPE-2394: If requested, avoid matching fields that cover any of the
-+
+    # intents that are to be excluded.
-+
+    if exclude_intents is not None:
-+
+        intent_fields = [f for f in intent_fields if f.intents.isdisjoint(set(exclude_intents.split(',')))]
-270
-271
+    # contains fields of this intent that also have a companion Tsys scan
-272
+    intent_fields_with_tsys = [f for f in intent_fields if 'ATMOSPHERE' in f.intents]
-273
-274
+    # contains fields without a companion Tsys scan. These might be science
-275
+    # mosaics.
-276
+    intent_fields_without_tsys = [f for f in intent_fields if f not in intent_fields_with_tsys]
-277
-278
+    tsys_fields_for_mosaics = []
-279
+    if intent == 'TARGET':
-296
-297
+    r = {field.id for field in intent_fields_with_tsys}
-298
+    r.update({field.id for field in tsys_fields_for_mosaics})
-299
-300
+    # when field names are not unique, as is usually the case for science
-301
+    # mosaics, then we must reference the numeric field ID instead
-302
+    field_identifiers = utils.get_field_identifiers(ms)
-303
+    return {field_identifiers[i] for i in r}
-304
-305
--
+def get_calapplications(ms: MeasurementSet, tsys_table: str, calfrom_defaults: Dict, origin: callibrary.CalAppOrigin,
--
+                        spw_map: List, is_single_dish: bool) -> List[callibrary.CalApplication]:
-+
+def get_calapplications(ms: MeasurementSet, tsys_table: str, calfrom_defaults: dict, origin: callibrary.CalAppOrigin,
-+
+                        spw_map: list, is_single_dish: bool) -> list[callibrary.CalApplication]:
-308
+    """
-309
+    Get a list of CalApplications that apply a Tsys caltable to a measurement
-310
+    set using the gainfield mapping defined in CAS-12213.
-311
-312
+    Note: this function only provides the gainfield argument for the CalFrom
-313
+    constructor. Any other required CalFrom constructor arguments should be
-314
+    provided to this function via the calfrom_defaults parameter.
-315
--
+    :param ms: MeasurementSet to apply calibrations to
--
+    :param tsys_table: name of Tsys table
--
+    :param calfrom_defaults: dict of CalFrom constructor arguments
--
+    :param origin: CalOrigin for the created CalApplications
--
+    :param spw_map: Tsys SpW map
--
+    :param is_single_dish: boolean declaring if current MS is for Single-Dish
--
+    :return: list of CalApplications
-+
+    Args:
-+
+        ms: MeasurementSet to apply calibrations to.
-+
+        tsys_table: name of Tsys table.
-+
+        calfrom_defaults: dict of CalFrom constructor arguments.
-+
+        origin: CalOrigin for the created CalApplications.
-+
+        spw_map: Tsys SpW map.
-+
+        is_single_dish: boolean declaring if current MS is for Single-Dish.
-+
-+
+    Returns:
-+
+        List of CalApplications.
-326
+    """
-327
+    # Get the map of intent:gainfield
-328
+    soln_map = get_gainfield_map(ms, is_single_dish)
-329
-330
+    # Create the static dict of calfrom arguments. Only the 'gainfield' argument changes from calapp to calapp; the
-331
+    # other arguments remain unchanged.
-332
+    calfrom_args = dict(calfrom_defaults)
-333
+    calfrom_args['gaintable'] = tsys_table
-334
-335
+    # get the mapping of field ID to unambiguous identifier for more user friendly logs

pipeline

Commits

Jira Issues

Add shortcut