Commits

Benjamin Bean authored d3e9dab1cf1
CAS-6692 remove all the temporary workarounds necessary until CAS-13683 is done
No tags

casatasks/src/private/imagerhelpers/summary_minor.py

Modified
43 43 getAll(summary_key): returns the list of all values for the given summary key
44 44 getDict(calc_iterdone_deltas, keep_startvals): to get the iterDone stat for iterations across all channels
45 45 getOutlierField(fieldid): to get a new SummaryMinor instance the contents of an outlier field other than field 0
46 46
47 47 Extends the python dictionary interface (try the command "help(dict)" for more information on builtin python dicts).
48 48
49 49 Note that this class will be saved as a plain python dict when saved with methods like pickle.dump() or numpy.save().
50 50 This is done to prevent issues when loading later, when this class might not be available to the python interpretter."""
51 51 # 0 1 2 3 4 5 6 7 8 9 10 11 "No Mask" 12 13 14 15 16 17 18
52 52 rowDescriptionsOldOrder = ["iterDone", "peakRes", "modelFlux", "cycleThresh", "mapperId", "chan", "pol", "cycleStartIters", "startIterDone", "startPeakRes", "startModelFlux", "startPeakResNM", "peakResNM", "masksum", "mpiServer", "peakMem", "runtime", "immod", "stopCode"]
53 - rowDescriptions13683 = ["iterDone", "peakRes", "modelFlux", "cycleThresh", "immod", "chan"]
54 53 rowDescriptions = ["startIterDone", "iterDone", "startPeakRes", "peakRes", "startModelFlux", "modelFlux", "startPeakResNM", "peakResNM", "cycleThresh", "mapperId", "cycleStartIters", "masksum", "mpiServer", "peakMem", "runtime", "immod", "stopCode", "chan", "pol"]
55 54 rowStartDescs = ["startIterDone", "startPeakRes", "startModelFlux", "startPeakResNM"]
56 55
57 56 def __init__(self, summaryminor_matrix, summaryminor_dict = None):
58 57 self.summaryminor_matrix = summaryminor_matrix
59 58 self.singleFieldMatrix = summaryminor_matrix
60 59 self.fieldIds = SummaryMinor._getFieldIds(summaryminor_matrix)
61 60 if len(self.fieldIds) > 1:
62 61 self.singleFieldMatrix = SummaryMinor._getSingleFieldMatrix(summaryminor_matrix, self.fieldIds[0])
63 62 if (len(self.fieldIds) > 1):
119 118 matrixOut[rowOut,colOut] = matrixIn[rowIn,colIn]
120 119 maxRowOut = max(rowOut, maxRowOut)
121 120 maxColOut = colOut
122 121 colOut += 1
123 122
124 123 return matrixOut
125 124
126 125 def getMatrix(self, fieldId=-1):
127 126 """Returns the original numpy.ndarray matrix.
128 127 fieldId: None for the original matrix, -1 for the matrix used to populate this dictionary, or choose another outlier fields
129 - Index 0: row (see this.rowDescriptionsOldOrder/rowDescriptions13683)
128 + Index 0: row (see this.rowDescriptionsOldOrder)
130 129 Index 1: values for all the minor cycles and outlier fields"""
131 130 if (fieldId == None):
132 131 return self.summaryminor_matrix
133 132 if (fieldId == -1):
134 133 return self.singleFieldMatrix
135 134 return SummaryMinor._getSingleFieldMatrix(self.summaryminor_matrix, fieldId)
136 135
137 136 def getAll(self, summary_key):
138 137 """Return the numpy matrix of all values for the given key (eg "chan")"""
139 138 idx = self.getRowDescriptionsOldOrder().index(summary_key)
140 139 return self.getMatrix()[idx,:]
141 140
142 - def useSmallSummaryminor(ignored_parameter=None):
143 - """Temporary CAS-13683 workaround"""
144 - if ('USE_SMALL_SUMMARYMINOR' in os.environ):
145 - uss = os.environ['USE_SMALL_SUMMARYMINOR'].lower()
146 - if (uss == "true"):
147 - return True
148 - return False
149 -
150 - def _getRowDescriptionsOldOrder(useSmallSummaryminor):
151 - """Temporary CAS-13683 workaround"""
152 - if (useSmallSummaryminor):
153 - return SummaryMinor.rowDescriptions13683
154 - return SummaryMinor.rowDescriptionsOldOrder
155 -
156 141 def getRowDescriptionsOldOrder():
157 - return SummaryMinor._getRowDescriptionsOldOrder(SummaryMinor.useSmallSummaryminor())
158 -
159 - def _getRowDescriptions(useSmallSummaryminor):
160 - """Temporary CAS-13683 workaround"""
161 - ret = SummaryMinor.rowDescriptions
162 - availRows = SummaryMinor._getRowDescriptionsOldOrder(useSmallSummaryminor)
163 - ret = list(filter(lambda x: x in availRows, ret))
164 - return ret
142 + return SummaryMinor.rowDescriptionsOldOrder
165 143
166 144 def getRowDescriptions():
167 - return SummaryMinor._getRowDescriptions(SummaryMinor.useSmallSummaryminor())
168 -
169 - def _getRowStartDescs(useSmallSummaryminor):
170 - """Temporary CAS-13683 workaround"""
171 - ret = SummaryMinor.rowStartDescs
172 - availRows = SummaryMinor._getRowDescriptionsOldOrder(useSmallSummaryminor)
173 - ret = list(filter(lambda x: x in availRows, ret))
174 - return ret
145 + return SummaryMinor.rowDescriptions
175 146
176 147 def getRowStartDescs():
177 - return SummaryMinor._getRowStartDescs(SummaryMinor.useSmallSummaryminor())
148 + return SummaryMinor.rowStartDescs
178 149
179 150 def indexMinorCycleSummaryBySubimage(matrix):
180 151 """Re-indexes matrix from [row,column] to [channel,polarity,row,cycle]."""
181 152 # get some properties of the summary_minor matrix
182 153 nrows = matrix.shape[0]
183 154 ncols = matrix.shape[1]
184 - uss = SummaryMinor.useSmallSummaryminor() # Temporary CAS-13683 workaround
185 155 import sys
186 - oldChanIdx = SummaryMinor._getRowDescriptionsOldOrder(uss).index("chan")
187 - newChanIdx = SummaryMinor._getRowDescriptions(uss).index("chan")
188 - if not uss:
189 - oldPolIdx = SummaryMinor._getRowDescriptionsOldOrder(uss).index("pol")
190 - newPolIdx = SummaryMinor._getRowDescriptions(uss).index("pol")
156 + oldChanIdx = SummaryMinor.getRowDescriptionsOldOrder().index("chan")
157 + newChanIdx = SummaryMinor.getRowDescriptions().index("chan")
158 + oldPolIdx = SummaryMinor.getRowDescriptionsOldOrder().index("pol")
159 + newPolIdx = SummaryMinor.getRowDescriptions().index("pol")
191 160 chans = list(np.sort(np.unique(matrix[oldChanIdx])))
192 161 chans = [int(x) for x in chans]
193 - if uss:
194 - pols = [0]
195 - else:
196 - pols = list(np.sort(np.unique(matrix[oldPolIdx])))
197 - pols = [int(x) for x in pols]
162 + pols = list(np.sort(np.unique(matrix[oldPolIdx])))
163 + pols = [int(x) for x in pols]
198 164 ncycles = 0
199 165 if len(chans) > 0 and len(pols) > 0:
200 166 ncycles = int( ncols / (len(chans)*len(pols)) )
201 - if uss:
202 - try:
203 - from casampi.MPIEnvironment import MPIEnvironment
204 - if MPIEnvironment.is_mpi_enabled:
205 - # This is necessary because we may have an odd number of "channels" due to each process getting only a subchunk.
206 - # Example:
207 - # Process 1 gets polarities 0-1, process 2 gets polarity 2
208 - # Each of them assigns channel id = chan + pol * nsubpols
209 - # Process 1 assigns channel ids [0,2], Process 2 assigns channel id 0.
210 - # This hack is not needed when not using a small summary minor because we have the extra knowledge of the polarities, instead of mapping polarities + channels onto chunks.
211 - chanslist = matrix[oldChanIdx].tolist()
212 - for chan in chans:
213 - singlechan_occurances = list(filter(lambda x: x == chan, chanslist))
214 - ncycles = max(ncycles, len(singlechan_occurances))
215 - except ModuleNotFoundError as e:
216 - raise
217 167
218 168 # ret is the return dictionary[chans][pols][rows][cycles]
219 169 # cummulativeCnt counts how many cols we've read for each channel/polarity/row
220 - ret = {desc:[0]*ncycles for desc in SummaryMinor._getRowDescriptions(uss)}
170 + ret = {desc:[0]*ncycles for desc in SummaryMinor.getRowDescriptions()}
221 171 # channel and polarity information is in the indexing, don't need to add it to the return dict
222 172 for desc in ["chan", "pol"]:
223 173 if desc in ret:
224 174 del ret[desc]
225 175 ret = {pol:copy.deepcopy(ret) for pol in pols}
226 176 ret = {chan:copy.deepcopy(ret) for chan in chans}
227 177 cummulativeCnt = copy.deepcopy(ret) # copy ret's structure
228 178
229 179 # reindex based on subimage index (aka chan/pol index)
230 - for desc in SummaryMinor._getRowDescriptions(uss):
180 + for desc in SummaryMinor.getRowDescriptions():
231 181 if desc in ["chan", "pol"]:
232 182 # channel and polarity information is in the indexing, don't need to add it to the return dict
233 183 continue
234 - oldRowIdx = SummaryMinor._getRowDescriptionsOldOrder(uss).index(desc)
184 + oldRowIdx = SummaryMinor.getRowDescriptionsOldOrder().index(desc)
235 185 for colIdx in range(ncols):
236 186 chan = int(matrix[oldChanIdx][colIdx])
237 - if (uss):
238 - pol = 0
239 - else:
240 - pol = int(matrix[oldPolIdx][colIdx])
187 + pol = int(matrix[oldPolIdx][colIdx])
241 188 val = matrix[oldRowIdx][colIdx]
242 189 cummulativeCol = int(cummulativeCnt[chan][pol][desc][0]) # const 0: cummulativeCnt doesn't make use of 'cycle' index from copied ret structure
243 190 ret[chan][pol][desc][cummulativeCol] = val
244 191 cummulativeCnt[chan][pol][desc][0] += 1
245 192
246 193 return ret
247 194
248 195 def _getPerCycleDict(summaryminor_dict, calc_iterdone_deltas=None, keep_startvals=None):
249 196 calc_iterdone_deltas = True if (calc_iterdone_deltas == None) else calc_iterdone_deltas
250 197 keep_startvals = True if (keep_startvals == None) else keep_startvals
251 198 ret = summaryminor_dict
252 - uss = SummaryMinor.useSmallSummaryminor() # Temporary CAS-13683 workaround
253 - availRows = SummaryMinor._getRowDescriptionsOldOrder(uss)
199 + availRows = SummaryMinor.getRowDescriptionsOldOrder()
254 200
255 201 if (calc_iterdone_deltas) and ("startIterDone" in availRows):
256 202 for chan in ret:
257 203 for pol in ret[chan]:
258 204 for cyc in range(len(ret[chan][pol]["startIterDone"])):
259 205 ret[chan][pol]["iterDone"][cyc] -= ret[chan][pol]["startIterDone"][cyc]
260 206 if not keep_startvals:
261 207 for chan in ret:
262 208 for pol in ret[chan]:
263 - for desc in SummaryMinor._getRowStartDescs(uss):
209 + for desc in SummaryMinor.getRowStartDescs():
264 210 del ret[chan][pol][desc]
265 211
266 212 return ret
267 213
268 214 def getOutlierField(self, fieldId):
269 215 """Get a new SummaryMinor instance for the given fieldId.
270 216
271 217 Example:
272 218 nchan_tot = 0
273 219 for field_id in summ_min.fieldIds:

Everything looks good. We'll let you know here if there's anything you should know about.

Add shortcut