Commits

Kristin Berry authored 516f270e389
PIPE-2094: Fix issue that was preventing stats_extractor results from being added to the pool and update structure of flagdata_percentage to not repeat the MS name
No tags

pipeline/infrastructure/renderer/stats_extractor.py

Modified
113 113 self.add_handler(plugin_class())
114 114 self.__plugins_loaded = True
115 115
116 116 # this is the list which will contain extracted values
117 117 extracted = []
118 118
119 119 # Process leaf results first
120 120 if isinstance(result, collections.abc.Iterable):
121 121 for r in result:
122 122 d = self.handle(r, context)
123 - union(extracted, d)
123 + extracted = union(extracted, d)
124 124
125 125 # process the group-level results.
126 126 for handler in self.__handlers:
127 127 if handler.is_handler_for(result):
128 128 LOG.debug('{} extracting stats results for {}'.format(handler.__class__.__name__,
129 129 result.__class__.__name__))
130 130 d = handler.handle(result, context)
131 - union(extracted, d)
131 + extracted = union(extracted, d)
132 132
133 133 return extracted
134 134
135 135
136 136 # default StatsExtractorRegistry initialization
137 137 registry = StatsExtractorRegistry()
138 138
139 139
140 140 class FlagDeterALMAResultsExtractor(StatsExtractor):
141 141 result_cls = FlagDeterALMAResults
157 157 for ms in flag_totals:
158 158 output_dict[ms] = {}
159 159 for reason in flag_totals[ms]:
160 160 for intent in flag_totals[ms][reason]:
161 161 if reason in reasons_to_export:
162 162 if "TOTAL" in intent:
163 163 new = float(flag_totals[ms][reason][intent][0])
164 164 total = float(flag_totals[ms][reason][intent][1])
165 165 percentage = new/total * 100
166 166 output_dict[ms][reason] = percentage
167 +
167 168 mous = context.get_oussid()
168 - ps = pstats.PipelineStatistics(name="flagdata_percentage",
169 - value=output_dict,
170 - longdesc="temporory value for testing",
171 - eb=ms,
172 - mous=mous,
173 - level=pstats.PipelineStatisticsLevel.EB)
174 - return ps
169 + longdescription = "dictionary giving percentage of data newly flagged by the following intents: online, shadow, qa0, qa2 before and template flagging agents"
170 + stats = []
171 + for ms in output_dict:
172 + ps = pstats.PipelineStatistics(name="flagdata_percentage",
173 + value=output_dict[ms],
174 + longdesc=longdescription,
175 + eb=ms,
176 + mous=mous,
177 + level=pstats.PipelineStatisticsLevel.EB)
178 + stats.append(ps)
179 + return stats
175 180
176 181
177 182 class FluxcalflagStatsExtractor(StatsExtractor):
178 183 result_cls = GfluxscaleflagResults
179 184 child_cls = None
180 185
181 186 def handle(self, result:GfluxscaleflagResults, context):
182 187 """
183 188 Args:
184 189 result: GfluxscaleflagResults object
185 190 """
186 191 summaries_by_name = {s['name']: s for s in result.cafresult.summaries}
187 192
188 193 num_flags_before = summaries_by_name['before']['flagged']
189 194
190 195 if 'after' in summaries_by_name:
191 196 num_flags_after = summaries_by_name['after']['flagged']
192 197 else:
193 198 num_flags_after = num_flags_before
194 199
195 200 ps = pstats.PipelineStatistics(name="fluxscaleflags",
196 - value=int(num_flags_after),
197 - longdesc="rows after",
198 - mous=context.get_oussid(),
199 - level=pstats.PipelineStatisticsLevel.MOUS)
201 + value=int(num_flags_after),
202 + longdesc="rows after",
203 + mous=context.get_oussid(),
204 + level=pstats.PipelineStatisticsLevel.EB)
200 205 return ps
201 206
202 207
203 -class ApplycalRegressionExtractor(StatsExtractor):
208 +class ApplycalExtractor(StatsExtractor):
204 209 """
205 210 Stats test result extractor for applycal tasks.
206 211 """
207 212
208 213 result_cls = ApplycalResults
209 214 child_cls = None
210 215 generating_task = IFApplycal
211 216
212 217 def handle(self, result: ApplycalResults, context):
213 218 """
214 219 Args:
215 220 result: ApplycalResults object
216 221
217 222 Returns:
218 223 OrderedDict[str, float]
219 224 """
220 225 summaries_by_name = {s['name']: s for s in result.summaries}
221 226 num_flags_after = summaries_by_name['applycal']['flagged']
222 227 ps = pstats.PipelineStatistics(name="applycal_flags",
223 - value=int(num_flags_after),
224 - longdesc="rows after",
225 - mous=context.get_oussid(),
226 - level=pstats.PipelineStatisticsLevel.MOUS)
228 + value=int(num_flags_after),
229 + longdesc="rows after",
230 + mous=context.get_oussid(),
231 + level=pstats.PipelineStatisticsLevel.MOUS)
227 232 return ps
228 233
229 234
230 235 def get_stats_from_results(context: Context) -> List[pstats.PipelineStatistics]:
231 236 """
232 237 Gathers all possible pipeline statistics from results.
233 238 """
234 239 unified = []
235 240 for results_proxy in context.results:
236 241 results = results_proxy.read()
237 - union(unified, registry.handle(results, context))
238 -
242 + unified = union(unified, registry.handle(results, context))
239 243 return unified
240 244
241 245
242 246 def union(lst: List, new: Union[pstats.PipelineStatistics, List[pstats.PipelineStatistics]]) -> List[pstats.PipelineStatistics]:
243 247 """
244 248 Combines lst which is always a list, with new,
245 249 which could be a list of PipelineStatistics objects
246 250 or an individual PipelineStatistics object.
247 251 """
248 252 union = copy.deepcopy(lst)
249 -
250 253 if isinstance(new, list):
251 254 for elt in new:
252 255 union.append(elt)
253 256 else:
254 257 union.append(new)
255 -
256 258 return union
257 259
258 260
259 261 def generate_stats(context) -> Dict:
260 262 """
261 263 Gathers statistics from the context and results and returns a representation
262 264 of them as a dict.
263 265 """
264 266 stats_collection = []
265 267

Everything looks good. We'll let you know here if there's anything you should know about.

Add shortcut