Commits
Benjamin Bean authored d3e9dab1cf1
43 43 | getAll(summary_key): returns the list of all values for the given summary key |
44 44 | getDict(calc_iterdone_deltas, keep_startvals): to get the iterDone stat for iterations across all channels |
45 45 | getOutlierField(fieldid): to get a new SummaryMinor instance the contents of an outlier field other than field 0 |
46 46 | |
47 47 | Extends the python dictionary interface (try the command "help(dict)" for more information on builtin python dicts). |
48 48 | |
49 49 | Note that this class will be saved as a plain python dict when saved with methods like pickle.dump() or numpy.save(). |
50 50 | This is done to prevent issues when loading later, when this class might not be available to the python interpretter.""" |
51 51 | # 0 1 2 3 4 5 6 7 8 9 10 11 "No Mask" 12 13 14 15 16 17 18 |
52 52 | rowDescriptionsOldOrder = ["iterDone", "peakRes", "modelFlux", "cycleThresh", "mapperId", "chan", "pol", "cycleStartIters", "startIterDone", "startPeakRes", "startModelFlux", "startPeakResNM", "peakResNM", "masksum", "mpiServer", "peakMem", "runtime", "immod", "stopCode"] |
53 - | rowDescriptions13683 = ["iterDone", "peakRes", "modelFlux", "cycleThresh", "immod", "chan"] |
54 53 | rowDescriptions = ["startIterDone", "iterDone", "startPeakRes", "peakRes", "startModelFlux", "modelFlux", "startPeakResNM", "peakResNM", "cycleThresh", "mapperId", "cycleStartIters", "masksum", "mpiServer", "peakMem", "runtime", "immod", "stopCode", "chan", "pol"] |
55 54 | rowStartDescs = ["startIterDone", "startPeakRes", "startModelFlux", "startPeakResNM"] |
56 55 | |
57 56 | def __init__(self, summaryminor_matrix, summaryminor_dict = None): |
58 57 | self.summaryminor_matrix = summaryminor_matrix |
59 58 | self.singleFieldMatrix = summaryminor_matrix |
60 59 | self.fieldIds = SummaryMinor._getFieldIds(summaryminor_matrix) |
61 60 | if len(self.fieldIds) > 1: |
62 61 | self.singleFieldMatrix = SummaryMinor._getSingleFieldMatrix(summaryminor_matrix, self.fieldIds[0]) |
63 62 | if (len(self.fieldIds) > 1): |
119 118 | matrixOut[rowOut,colOut] = matrixIn[rowIn,colIn] |
120 119 | maxRowOut = max(rowOut, maxRowOut) |
121 120 | maxColOut = colOut |
122 121 | colOut += 1 |
123 122 | |
124 123 | return matrixOut |
125 124 | |
126 125 | def getMatrix(self, fieldId=-1): |
127 126 | """Returns the original numpy.ndarray matrix. |
128 127 | fieldId: None for the original matrix, -1 for the matrix used to populate this dictionary, or choose another outlier fields |
129 - | Index 0: row (see this.rowDescriptionsOldOrder/rowDescriptions13683) |
128 + | Index 0: row (see this.rowDescriptionsOldOrder) |
130 129 | Index 1: values for all the minor cycles and outlier fields""" |
131 130 | if (fieldId == None): |
132 131 | return self.summaryminor_matrix |
133 132 | if (fieldId == -1): |
134 133 | return self.singleFieldMatrix |
135 134 | return SummaryMinor._getSingleFieldMatrix(self.summaryminor_matrix, fieldId) |
136 135 | |
137 136 | def getAll(self, summary_key): |
138 137 | """Return the numpy matrix of all values for the given key (eg "chan")""" |
139 138 | idx = self.getRowDescriptionsOldOrder().index(summary_key) |
140 139 | return self.getMatrix()[idx,:] |
141 140 | |
142 - | def useSmallSummaryminor(ignored_parameter=None): |
143 - | """Temporary CAS-13683 workaround""" |
144 - | if ('USE_SMALL_SUMMARYMINOR' in os.environ): |
145 - | uss = os.environ['USE_SMALL_SUMMARYMINOR'].lower() |
146 - | if (uss == "true"): |
147 - | return True |
148 - | return False |
149 - | |
150 - | def _getRowDescriptionsOldOrder(useSmallSummaryminor): |
151 - | """Temporary CAS-13683 workaround""" |
152 - | if (useSmallSummaryminor): |
153 - | return SummaryMinor.rowDescriptions13683 |
154 - | return SummaryMinor.rowDescriptionsOldOrder |
155 - | |
156 141 | def getRowDescriptionsOldOrder(): |
157 - | return SummaryMinor._getRowDescriptionsOldOrder(SummaryMinor.useSmallSummaryminor()) |
158 - | |
159 - | def _getRowDescriptions(useSmallSummaryminor): |
160 - | """Temporary CAS-13683 workaround""" |
161 - | ret = SummaryMinor.rowDescriptions |
162 - | availRows = SummaryMinor._getRowDescriptionsOldOrder(useSmallSummaryminor) |
163 - | ret = list(filter(lambda x: x in availRows, ret)) |
164 - | return ret |
142 + | return SummaryMinor.rowDescriptionsOldOrder |
165 143 | |
166 144 | def getRowDescriptions(): |
167 - | return SummaryMinor._getRowDescriptions(SummaryMinor.useSmallSummaryminor()) |
168 - | |
169 - | def _getRowStartDescs(useSmallSummaryminor): |
170 - | """Temporary CAS-13683 workaround""" |
171 - | ret = SummaryMinor.rowStartDescs |
172 - | availRows = SummaryMinor._getRowDescriptionsOldOrder(useSmallSummaryminor) |
173 - | ret = list(filter(lambda x: x in availRows, ret)) |
174 - | return ret |
145 + | return SummaryMinor.rowDescriptions |
175 146 | |
176 147 | def getRowStartDescs(): |
177 - | return SummaryMinor._getRowStartDescs(SummaryMinor.useSmallSummaryminor()) |
148 + | return SummaryMinor.rowStartDescs |
178 149 | |
179 150 | def indexMinorCycleSummaryBySubimage(matrix): |
180 151 | """Re-indexes matrix from [row,column] to [channel,polarity,row,cycle].""" |
181 152 | # get some properties of the summary_minor matrix |
182 153 | nrows = matrix.shape[0] |
183 154 | ncols = matrix.shape[1] |
184 - | uss = SummaryMinor.useSmallSummaryminor() # Temporary CAS-13683 workaround |
185 155 | import sys |
186 - | oldChanIdx = SummaryMinor._getRowDescriptionsOldOrder(uss).index("chan") |
187 - | newChanIdx = SummaryMinor._getRowDescriptions(uss).index("chan") |
188 - | if not uss: |
189 - | oldPolIdx = SummaryMinor._getRowDescriptionsOldOrder(uss).index("pol") |
190 - | newPolIdx = SummaryMinor._getRowDescriptions(uss).index("pol") |
156 + | oldChanIdx = SummaryMinor.getRowDescriptionsOldOrder().index("chan") |
157 + | newChanIdx = SummaryMinor.getRowDescriptions().index("chan") |
158 + | oldPolIdx = SummaryMinor.getRowDescriptionsOldOrder().index("pol") |
159 + | newPolIdx = SummaryMinor.getRowDescriptions().index("pol") |
191 160 | chans = list(np.sort(np.unique(matrix[oldChanIdx]))) |
192 161 | chans = [int(x) for x in chans] |
193 - | if uss: |
194 - | pols = [0] |
195 - | else: |
196 - | pols = list(np.sort(np.unique(matrix[oldPolIdx]))) |
197 - | pols = [int(x) for x in pols] |
162 + | pols = list(np.sort(np.unique(matrix[oldPolIdx]))) |
163 + | pols = [int(x) for x in pols] |
198 164 | ncycles = 0 |
199 165 | if len(chans) > 0 and len(pols) > 0: |
200 166 | ncycles = int( ncols / (len(chans)*len(pols)) ) |
201 - | if uss: |
202 - | try: |
203 - | from casampi.MPIEnvironment import MPIEnvironment |
204 - | if MPIEnvironment.is_mpi_enabled: |
205 - | # This is necessary because we may have an odd number of "channels" due to each process getting only a subchunk. |
206 - | # Example: |
207 - | # Process 1 gets polarities 0-1, process 2 gets polarity 2 |
208 - | # Each of them assigns channel id = chan + pol * nsubpols |
209 - | # Process 1 assigns channel ids [0,2], Process 2 assigns channel id 0. |
210 - | # This hack is not needed when not using a small summary minor because we have the extra knowledge of the polarities, instead of mapping polarities + channels onto chunks. |
211 - | chanslist = matrix[oldChanIdx].tolist() |
212 - | for chan in chans: |
213 - | singlechan_occurances = list(filter(lambda x: x == chan, chanslist)) |
214 - | ncycles = max(ncycles, len(singlechan_occurances)) |
215 - | except ModuleNotFoundError as e: |
216 - | raise |
217 167 | |
218 168 | # ret is the return dictionary[chans][pols][rows][cycles] |
219 169 | # cummulativeCnt counts how many cols we've read for each channel/polarity/row |
220 - | ret = {desc:[0]*ncycles for desc in SummaryMinor._getRowDescriptions(uss)} |
170 + | ret = {desc:[0]*ncycles for desc in SummaryMinor.getRowDescriptions()} |
221 171 | # channel and polarity information is in the indexing, don't need to add it to the return dict |
222 172 | for desc in ["chan", "pol"]: |
223 173 | if desc in ret: |
224 174 | del ret[desc] |
225 175 | ret = {pol:copy.deepcopy(ret) for pol in pols} |
226 176 | ret = {chan:copy.deepcopy(ret) for chan in chans} |
227 177 | cummulativeCnt = copy.deepcopy(ret) # copy ret's structure |
228 178 | |
229 179 | # reindex based on subimage index (aka chan/pol index) |
230 - | for desc in SummaryMinor._getRowDescriptions(uss): |
180 + | for desc in SummaryMinor.getRowDescriptions(): |
231 181 | if desc in ["chan", "pol"]: |
232 182 | # channel and polarity information is in the indexing, don't need to add it to the return dict |
233 183 | continue |
234 - | oldRowIdx = SummaryMinor._getRowDescriptionsOldOrder(uss).index(desc) |
184 + | oldRowIdx = SummaryMinor.getRowDescriptionsOldOrder().index(desc) |
235 185 | for colIdx in range(ncols): |
236 186 | chan = int(matrix[oldChanIdx][colIdx]) |
237 - | if (uss): |
238 - | pol = 0 |
239 - | else: |
240 - | pol = int(matrix[oldPolIdx][colIdx]) |
187 + | pol = int(matrix[oldPolIdx][colIdx]) |
241 188 | val = matrix[oldRowIdx][colIdx] |
242 189 | cummulativeCol = int(cummulativeCnt[chan][pol][desc][0]) # const 0: cummulativeCnt doesn't make use of 'cycle' index from copied ret structure |
243 190 | ret[chan][pol][desc][cummulativeCol] = val |
244 191 | cummulativeCnt[chan][pol][desc][0] += 1 |
245 192 | |
246 193 | return ret |
247 194 | |
248 195 | def _getPerCycleDict(summaryminor_dict, calc_iterdone_deltas=None, keep_startvals=None): |
249 196 | calc_iterdone_deltas = True if (calc_iterdone_deltas == None) else calc_iterdone_deltas |
250 197 | keep_startvals = True if (keep_startvals == None) else keep_startvals |
251 198 | ret = summaryminor_dict |
252 - | uss = SummaryMinor.useSmallSummaryminor() # Temporary CAS-13683 workaround |
253 - | availRows = SummaryMinor._getRowDescriptionsOldOrder(uss) |
199 + | availRows = SummaryMinor.getRowDescriptionsOldOrder() |
254 200 | |
255 201 | if (calc_iterdone_deltas) and ("startIterDone" in availRows): |
256 202 | for chan in ret: |
257 203 | for pol in ret[chan]: |
258 204 | for cyc in range(len(ret[chan][pol]["startIterDone"])): |
259 205 | ret[chan][pol]["iterDone"][cyc] -= ret[chan][pol]["startIterDone"][cyc] |
260 206 | if not keep_startvals: |
261 207 | for chan in ret: |
262 208 | for pol in ret[chan]: |
263 - | for desc in SummaryMinor._getRowStartDescs(uss): |
209 + | for desc in SummaryMinor.getRowStartDescs(): |
264 210 | del ret[chan][pol][desc] |
265 211 | |
266 212 | return ret |
267 213 | |
268 214 | def getOutlierField(self, fieldId): |
269 215 | """Get a new SummaryMinor instance for the given fieldId. |
270 216 | |
271 217 | Example: |
272 218 | nchan_tot = 0 |
273 219 | for field_id in summ_min.fieldIds: |