Commits

Takeshi Nakazato authored and Ville Suoranta committed 4984c9aab00 Merge
Pull request #832: CAS-14171

Merge in CASA/casa6 from CAS-14171 to master * commit '7dc0539c7526704f547cf4f6230febf1e5f5052a': refactoring Refactoring for readability Fail explicitly when exception occured during eval add warning comment on possible intermittent failure use f.tell() to get offset position CAS-14171 add unit test for bug fix CAS-14171 sort processSpwList_ CAS-14171 sort input collection for std::set_difference
No tags

casatasks/tests/tasks/test_task_sdatmcor.py

Modified
99 99 var_org = os.environ.get(var_name)
100 100 try:
101 101 yield var_org
102 102 finally:
103 103 if var_org is None:
104 104 os.environ.pop(var_name, None)
105 105 else:
106 106 os.environ[var_name] = var_org
107 107
108 108
109 +def get_end_pos_for(logfile: str) -> int:
110 + with open(logfile, 'r') as f:
111 + offset = 0
112 + from_end_of_file = os.SEEK_END
113 + f.seek(offset, from_end_of_file)
114 + return f.tell()
115 +
116 +
109 117 class test_sdatmcor(unittest.TestCase):
110 118 datapath = ctsys_resolve('measurementset/almasd')
111 119 infile = 'X320b_sel2.ms'
112 120 outfile = infile + '.atmcor'
113 121 caltable = infile + '.k2jycal'
114 122
115 123 local_unit_test = False
116 124
125 + casalog_seek_pos = 0
126 +
117 127 def setUp(self):
118 128 # default Args
119 129 self.args = {
120 130 'infile': self.infile,
121 131 'datacolumn': 'data',
122 132 'outfile': self.outfile,
123 133 }
124 134
125 135 smart_remove(self.infile)
126 136 smart_remove(self.outfile)
127 137 smart_remove(self.caltable)
128 138 shutil.copytree(os.path.join(self.datapath, self.infile), self.infile)
129 139
140 + self.casalog_seek_pos = get_end_pos_for(casalog.logfile())
141 +
130 142 def tearDown(self):
131 143 smart_remove(self.infile)
132 144 smart_remove(self.outfile)
133 145 smart_remove(self.caltable)
134 146
135 147 def skip_omp_test_if_darwin(self):
136 148 sysname = os.uname()[0]
137 149 if sysname == 'Darwin':
138 150 self.skipTest('Skip OpenMP tests on macOS')
139 151 return
212 224 if not on_source_only:
213 225 # OFF_SOURCE data should not be touched
214 226 mask0 = np.logical_and(stateids_before != 14, stateids_before != 84)
215 227 data_off_before = data_before[:, :, mask0]
216 228 mask1 = np.logical_and(stateids_after != 14, stateids_after != 84)
217 229 data_off_after = data_after[:, :, mask1]
218 230 self.assertTrue(np.all(data_off_before == data_off_after))
219 231 else:
220 232 self.assertTrue(np.all(data_after == data_before))
221 233
234 + def __check_casalog_spw_corrected(self, striplog: list, spwprocess: dict):
235 + """Test if casalog contains proper log message for list of corrected spws.
236 +
237 + Expected log message is,
238 +
239 + "processspw (input) = [XX, YY, ...]",
240 +
241 + where [XX, YY, ...] is a list of corrected spw ids.
242 +
243 + Args:
244 + striplog: Log message
245 + spwprocess: Dictionary describing input parameter
246 + """
247 + log_corrected = [line for line in striplog if line.startswith('processspw (input) = ')]
248 + spw_corrected = sorted([k for k, v in spwprocess.items() if v])
249 + # corrected spws must be non-zero
250 + self.assertGreater(len(spw_corrected), 0)
251 + # this log message must present
252 + self.assertEqual(len(log_corrected), 1)
253 + # extract list of spws from the log, and compare with expected list
254 + try:
255 + # the string to be evaluated should be a list like '[17, 19, 21, 23]'
256 + spw_list_str = log_corrected[0].split('=')[1].strip()
257 + spw_corrected_from_log = eval(spw_list_str)
258 + except Exception as e:
259 + print(str(e))
260 + self.fail(f'Unexpected log format: {spw_list_str}')
261 + self.assertEqual(spw_corrected_from_log, spw_corrected)
262 +
263 + def __check_casalog_spw_not_corrected(self, striplog: list, spwprocess: dict):
264 + """Test if casalog contains proper log message for list of *not* corrected spws.
265 +
266 + Expected log message is either,
267 +
268 + "SPWs XX YY ... are output but not corrected", or
269 + "SPW XX is output but not corrected",
270 +
271 + where XX YY ... or XX are a list of *not* corrected spw ids.
272 +
273 + Args:
274 + striplog: Log message
275 + spwprocess: Dictionary to describe spw selection
276 + """
277 + log_not_corrected = [line for line in striplog if line.endswith(' output but not corrected')]
278 + spw_not_corrected = sorted([k for k, v in spwprocess.items() if not v])
279 + if len(spw_not_corrected) > 0:
280 + # not corrected spws present, so log message must present
281 + self.assertEqual(len(log_not_corrected), 1)
282 + # extract list of spws from the log, and compare with expected list
283 + spw_not_corrected_from_log = [int(v) for v in log_not_corrected[0].split()[1:-5]]
284 + self.assertEqual(spw_not_corrected_from_log, spw_not_corrected)
285 + else:
286 + # all the spws are corrected, so log message must not present
287 + self.assertEqual(len(log_not_corrected), 0)
288 +
289 + def _check_casalog(self, spwprocess: dict):
290 + """Test if casalog contains expected log messages.
291 +
292 + See CAS-14171 for the purpose of the test.
293 +
294 + Args:
295 + spwprocess: Dictionary to describe spw selection.
296 + Keys are the list of spws selected (outputspw)
297 + while values indicate whether each spw is
298 + corrected (True) or not (False).
299 + """
300 + # WARNING
301 + # Current code may result in intermittent and non-deterministic test failures,
302 + # in case the expected log lines have not yet been flushed.
303 + # If such failures occur, please consider to make it possible to: casalog.flush()
304 + with open(casalog.logfile(), 'r') as f:
305 + f.seek(self.casalog_seek_pos, 0)
306 + log = f.read()
307 + lines = log.split('\n')
308 + # strip meta data, only keep body of the log
309 + striplog = [line.split('\t')[-1] for line in lines]
310 + self.assertGreater(len(log), 0)
311 + self.__check_casalog_spw_corrected(striplog, spwprocess)
312 + self.__check_casalog_spw_not_corrected(striplog, spwprocess)
313 +
222 314 def check_result(self, spwprocess, on_source_only=False):
223 315 """Check Result.
224 316
225 317 Args:
226 318 spwprocess (dict): key is spw id, value is whether or not the spw is processed
227 319 """
228 320 # outfile should exist
229 321 self.assertTrue(os.path.exists(self.outfile))
230 322
231 323 for spw in [19, 23]:
232 324 is_selected = spw in spwprocess
233 325 is_processed = spwprocess.get(spw, False)
234 326 self._check_result_spw(spw, is_selected, is_processed, on_source_only)
235 327
236 328 # test OpenMP related stuff
237 329 self.assertEqual(casalog.ompGetNumThreads(), OMP_NUM_THREADS_INITIAL)
238 330
331 + # test casalog messages
332 + self._check_casalog(spwprocess)
333 +
239 334 def test_sdatmcor_normal(self):
240 335 """Test normal usage of sdatmcor."""
241 336 sdatmcor(infile=self.infile, outfile=self.outfile, datacolumn='data')
242 337 self.check_result({19: True, 23: True})
243 338
244 339 def test_sdatmcor_explicit_atmtype(self):
245 340 """Test specifying atmtype explicitly."""
246 341 sdatmcor(infile=self.infile, outfile=self.outfile, datacolumn='data', atmtype=2)
247 342 self.check_result({19: True, 23: True})
248 343

Everything looks good. We'll let you know here if there's anything you should know about.

Add shortcut