casa / casa6 / 4d704b8cd9d

Commits

Benjamin Bean authored 4d704b8cd9d 28 Jan 2022

Revert "CAS-6692 remove all the temporary workarounds necessary until CAS-13683 is done, recommit after Urvashi's and Darrel's suggested changes"

This reverts commit bdc483eaa42aa3401cfa323023dfb6b76cb27a0f.

casatasks/src/private/imagerhelpers/summary_minor.py

Modified

-34
+            itersDone = field0[chan][stoke]['iterDone'][ncycles/2]
-35
-36
+. To get the available minor cycle summary statistics:
-37
+            field0 = summ_min[0]
-38
+            chan0 = field0.keys()[0]
-39
+            stoke0 = field0[chan0].keys()[0]
-40
+            availSummStats = field0[field0][stoke0].keys()
-41
+    """
-42
+    #                           0           1          2            3              4          5       6      7                  8                9               10                11 "No Mask"      12           13         14           15         16         17              18
-43
+    rowDescriptionsOldOrder = ["iterDone", "peakRes", "modelFlux", "cycleThresh", "deconId", "chan", "stoke", "cycleStartIters", "startIterDone", "startPeakRes", "startModelFlux", "startPeakResNM", "peakResNM", "masksum", "mpiServer", "peakMem", "runtime", "multifieldId", "stopCode"]
-+
+    rowDescriptions13683    = ["iterDone", "peakRes", "modelFlux", "cycleThresh", "deconId", "chan"]
-45
+    # rowDescriptions does not include {"multifieldId", "chan", "stoke", "deconId"}, and so the returned dictionary will not include those values in the summary keys
-46
+    rowDescriptions         = ["startIterDone", "iterDone", "startPeakRes", "peakRes", "startModelFlux", "modelFlux", "startPeakResNM", "peakResNM", "cycleThresh", "cycleStartIters", "masksum", "mpiServer", "peakMem", "runtime", "stopCode"]
-47
+    rowStartDescs           = ["startIterDone",             "startPeakRes",            "startModelFlux",              "startPeakResNM"]
-48
-49
+    def convertMatrix(summaryminor_matrix, calc_iterdone_deltas=None, keep_startvals=None):
-50
+        # casalog.post(summaryminor_matrix, "SEVERE")
-51
+        ret = {}
-52
-53
+        # edge case: no iterations were done (eg threshold < model flux)
-54
+        if summaryminor_matrix.shape[1] == 0:
-111
+                continue
-112
+            for rowIn in range(nrowsIn):
-113
+                rowOut = rowIn
-114
+                matrixOut[rowOut,colOut] = matrixIn[rowIn,colIn]
-115
+                maxRowOut = max(rowOut, maxRowOut)
-116
+            maxColOut = colOut
-117
+            colOut += 1
-118
-119
+        return matrixOut
-120
--
+    def getRowDescriptionsOldOrder():
-+
+    def useSmallSummaryminor(ignored_parameter=None):
-+
+        """Temporary CAS-13683 workaround"""
-+
+        if ('USE_SMALL_SUMMARYMINOR' in os.environ):
-+
+            uss = os.environ['USE_SMALL_SUMMARYMINOR'].lower()
-+
+            if (uss == "true"):
-+
+                return True
-+
+        return False
-+
-+
+    def _getRowDescriptionsOldOrder(useSmallSummaryminor):
-+
+        """Temporary CAS-13683 workaround"""
-+
+        if (useSmallSummaryminor):
-+
+            return SummaryMinor.rowDescriptions13683
-133
+        return SummaryMinor.rowDescriptionsOldOrder
-134
-+
+    def getRowDescriptionsOldOrder():
-+
+        """ Retrieves brief descriptions of the available minor cycle summary rows, in the old (matrix) order. """
-+
+        return SummaryMinor._getRowDescriptionsOldOrder(SummaryMinor.useSmallSummaryminor())
-+
-+
+    def _getRowDescriptions(useSmallSummaryminor):
-+
+        """Temporary CAS-13683 workaround"""
-+
+        ret = SummaryMinor.rowDescriptions
-+
+        availRows = SummaryMinor._getRowDescriptionsOldOrder(useSmallSummaryminor)
-+
+        ret = list(filter(lambda x: x in availRows, ret))
-+
+        return ret
-+
-146
+    def getRowDescriptions():
--
+        return SummaryMinor.rowDescriptions
-+
+        """ Retrieves brief descriptions of the available minor cycle summary rows """
-+
+        return SummaryMinor._getRowDescriptions(SummaryMinor.useSmallSummaryminor())
-+
-+
+    def _getRowStartDescs(useSmallSummaryminor):
-+
+        """Temporary CAS-13683 workaround"""
-+
+        ret = SummaryMinor.rowStartDescs
-+
+        availRows = SummaryMinor._getRowDescriptionsOldOrder(useSmallSummaryminor)
-+
+        ret = list(filter(lambda x: x in availRows, ret))
-+
+        return ret
-156
-157
+    def getRowStartDescs():
--
+        return SummaryMinor.rowStartDescs
-+
+        """ Retrieves abreviated names of the available minor cycle summary "start" rows.
-+
-+
+        These are the rows that catalog the values at the beggining of a minor cycle (pre-deconvolution). """
-+
+        return SummaryMinor._getRowStartDescs(SummaryMinor.useSmallSummaryminor())
-162
-163
+    def indexMinorCycleSummaryBySubimage(matrix):
-164
+        """ Re-indexes matrix from [row,column] to [channel,stokes,row,cycle].
-165
-166
+        Param matrix: the original matrix to convert.
-167
+        """
-168
+        # get some properties of the summary_minor matrix
-169
+        nrows = matrix.shape[0]
-170
+        ncols = matrix.shape[1]
-+
+        uss = SummaryMinor.useSmallSummaryminor() # Temporary CAS-13683 workaround
-172
+        import sys
-173
+        oldChanIdx = SummaryMinor.getRowDescriptionsOldOrder().index("chan")
--
+        oldStokeIdx  = SummaryMinor.getRowDescriptionsOldOrder().index("stoke")
-+
+        if not uss:
-+
+            oldStokeIdx  = SummaryMinor.getRowDescriptionsOldOrder().index("stoke")
-176
+        chans = list(np.sort(np.unique(matrix[oldChanIdx])))
-177
+        chans = [int(x) for x in chans]
--
+        stokes = list(np.sort(np.unique(matrix[oldStokeIdx])))
--
+        stokes = [int(x) for x in stokes]
-+
+        if uss:
-+
+            stokes = [0]
-+
+        else:
-+
+            stokes = list(np.sort(np.unique(matrix[oldStokeIdx])))
-+
+            stokes = [int(x) for x in stokes]
-183
+        ncycles = 0
-184
+        if len(chans) > 0 and len(stokes) > 0:
-185
+            ncycles = int( ncols / (len(chans)*len(stokes)) )
-+
+            if uss:
-+
+                try:
-+
+                    from casampi.MPIEnvironment import MPIEnvironment
-+
+                    if MPIEnvironment.is_mpi_enabled:
-+
+                        # This is necessary because we may have an odd number of "channels" due to each process getting only a subchunk.
-+
+                        # Example:
-+
+                        #     Process 1 gets stokes 0-1, process 2 gets stokes 2
-+
+                        #     Each of them assigns channel id = chan + stoke * nsubstokes
-+
+                        #     Process 1 assigns channel ids [0,2], Process 2 assigns channel id 0.
-+
+                        # This hack is not needed when not using a small summary minor because we have the extra knowledge of the stokes, instead of mapping stokes + channels onto chunks.
-+
+                        chanslist = matrix[oldChanIdx].tolist()
-+
+                        for chan in chans:
-+
+                            singlechan_occurances = list(filter(lambda x: x == chan, chanslist))
-+
+                            ncycles = max(ncycles, len(singlechan_occurances))
-+
+                except ModuleNotFoundError as e:
-+
+                    raise
-202
-203
+        # ret is the return dictionary[chans][stokes][rows][cycles]
-204
+        # cummulativeCnt counts how many cols we've read for each channel/stokes/row
-205
+        ret = {desc:[0]*ncycles for desc in SummaryMinor.getRowDescriptions()}
-206
+        ret = {stoke:copy.deepcopy(ret) for stoke in stokes}
-207
+        ret = {chan:copy.deepcopy(ret) for chan in chans}
-208
+        cummulativeCnt = copy.deepcopy(ret) # copy ret's structure
-209
-210
+        # reindex based on subimage index (aka chan/stoke index)
-211
+        for desc in SummaryMinor.getRowDescriptions():
-212
+            oldRowIdx = SummaryMinor.getRowDescriptionsOldOrder().index(desc)
-213
+            for colIdx in range(ncols):
-214
+                chan = int(matrix[oldChanIdx][colIdx])
--
+                stoke = int(matrix[oldStokeIdx][colIdx])
-+
+                if uss:
-+
+                    stoke = 0
-+
+                else:
-+
+                    stoke = int(matrix[oldStokeIdx][colIdx])
-219
+                val = matrix[oldRowIdx][colIdx]
-220
+                cummulativeCol = int(cummulativeCnt[chan][stoke][desc][0]) # const 0: cummulativeCnt doesn't make use of 'cycle' index from copied ret structure
-221
+                ret[chan][stoke][desc][cummulativeCol] = val
-222
+                cummulativeCnt[chan][stoke][desc][0] += 1
-223
-224
+        return ret
-225
-226
+    def _getPerCycleDict(summaryminor_dict, calc_iterdone_deltas=None, keep_startvals=None):
-227
+        calc_iterdone_deltas = True if (calc_iterdone_deltas == None) else calc_iterdone_deltas
-228
+        keep_startvals       = True if (keep_startvals == None)       else keep_startvals

casa6

Commits

Jira Issues

Add shortcut