Commits
Sandra M Castro authored b440fd015b6
182 182 | aflocal.parseagentparameters(agentUnflag) |
183 183 | aflocal.init() |
184 184 | aflocal.run(writeflags=True) |
185 185 | aflocal.done() |
186 186 | |
187 187 | class test_manual(test_base): |
188 188 | '''Test manual selections''' |
189 189 | |
190 190 | def setUp(self): |
191 191 | self.setUp_multi() |
192 - | |
192 + | self.filename = '' |
193 + | |
194 + | def tearDown(self) -> None: |
195 + | if os.path.exists(self.filename): |
196 + | os.system('rm -rf '+self.filename) |
197 + | |
198 + | |
199 + | def tearDownClass(cls) -> None: |
200 + | shutil.rmtree('multiobs.ms', ignore_errors=True) |
201 + | shutil.rmtree('ngc5921.ms', ignore_errors=True) |
202 + | shutil.rmtree('ngc5921.ms.flagversions', ignore_errors=True) |
203 + | |
193 204 | def test_observation(self): |
194 205 | myinput = "observation='1'" |
195 - | filename = create_input(myinput) |
206 + | self.filename = create_input(myinput) |
196 207 | |
197 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False, |
208 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=False, |
198 209 | flagbackup=False) |
199 210 | check_eq(flagdata(vis=self.vis, mode='summary'), 2882778, 28500) |
200 211 | |
201 212 | def test_compatibility(self): |
202 213 | myinput = "observation='1' mode='manualflag'" |
203 - | filename = create_input(myinput) |
204 - | |
205 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False, |
214 + | self.filename = create_input(myinput) |
215 + | |
216 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=False, |
206 217 | flagbackup=False) |
207 218 | check_eq(flagdata(vis=self.vis, mode='summary'), 2882778, 28500) |
208 219 | |
209 220 | def test_autocorr(self): |
210 221 | '''flagcmd: autocorr=True''' |
211 222 | self.setUp_ngc5921() |
212 223 | flagcmd(vis=self.vis, inpmode='list', inpfile=['autocorr=True'], action='apply') |
213 224 | res = flagdata(vis=self.vis, mode='summary') |
214 225 | self.assertEqual(res['flagged'], 203994, 'Should flag only the auto-correlations') |
215 226 | |
216 - | |
217 227 | class test_alma(test_base): |
218 228 | # Test various selections for alma data |
219 229 | |
220 230 | def setUp(self): |
221 231 | # self.setUp_flagdatatest_alma() |
222 232 | self.setUp_alma_ms() |
223 - | |
233 + | self.filename = '' |
234 + | |
235 + | def tearDown(self) -> None: |
236 + | if os.path.exists(self.filename): |
237 + | os.system('rm -rf '+self.filename) |
238 + | |
239 + | |
240 + | def tearDownClass(cls) -> None: |
241 + | shutil.rmtree('uid___A002_X30a93d_X43e_small.ms', ignore_errors=True) |
242 + | |
224 243 | def test_intent(self): |
225 244 | '''flagcmd: test scan intent selection''' |
226 245 | |
227 246 | myinput = "intent='CAL*POINT*'\n"\ |
228 247 | "#scan=3,4" |
229 - | filename = create_input(myinput) |
248 + | self.filename = create_input(myinput) |
230 249 | |
231 250 | # flag POINTING CALIBRATION scans and ignore comment line |
232 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False, |
251 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=False, |
233 252 | flagbackup=False) |
234 253 | # check_eq(flagdata(vis=self.vis,mode='summary', antenna='2'), 377280, 26200) |
235 254 | res = flagdata(vis=self.vis,mode='summary') |
236 255 | # self.assertEqual(res['scan']['1']['flagged'], 80184, 'Only scan 1 should be flagged') |
237 256 | # self.assertEqual(res['scan']['4']['flagged'], 0, 'Scan 4 should not be flagged') |
238 257 | self.assertEqual(res['scan']['1']['flagged'], 192416.0) |
239 258 | |
240 259 | def test_cmd(self): |
241 260 | '''flagcmd: inpmode=list with empty parameter''' |
242 261 | |
269 288 | self.assertEqual(ret['flagged'], 498) |
270 289 | self.assertEqual(ret['correlation']['I']['flagged'], 498) |
271 290 | self.assertEqual(ret['correlation']['XX']['flagged'], 0) |
272 291 | self.assertEqual(ret['correlation']['YY']['flagged'], 0) |
273 292 | |
274 293 | |
275 294 | class test_unapply(test_base): |
276 295 | # Action unapply |
277 296 | def setUp(self): |
278 297 | self.setUp_ngc5921() |
279 - | |
298 + | self.filename = '' |
299 + | |
300 + | def tearDown(self) -> None: |
301 + | if os.path.exists(self.filename): |
302 + | os.system('rm -rf '+self.filename) |
303 + | |
304 + | |
305 + | def tearDownClass(cls) -> None: |
306 + | shutil.rmtree('ngc5921.ms', ignore_errors=True) |
307 + | shutil.rmtree('ngc5921.ms.flagversions', ignore_errors=True) |
308 + | |
280 309 | def test_unsupported_unapply(self): |
281 310 | '''flagcmd: raise exception from inpmode=list and unapply''' |
282 311 | passes = False |
283 312 | try: |
284 313 | self.assertFalse(flagcmd( vis=self.vis, action='unapply', inpmode='list', |
285 314 | inpfile=["spw='0' reason='MANUAL'"]) ) |
286 315 | # CASA6 throws an except, CASA5 returns False |
287 316 | # only CASA6 gets here, but that's OK |
288 317 | passes = True |
289 318 | except ValueError: |
293 322 | |
294 323 | self.assertTrue(passes) |
295 324 | |
296 325 | def test_utfcrop(self): |
297 326 | '''flagcmd: unapply tfcrop agent''' |
298 327 | # Remove any cmd from table |
299 328 | flagcmd(vis=self.vis, action='clear', clearall=True) |
300 329 | |
301 330 | # Flag using manual agent |
302 331 | myinput = "scan='1'" |
303 - | filename = create_input(myinput) |
304 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True) |
332 + | self.filename = create_input(myinput) |
333 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=True) |
305 334 | |
306 335 | # Flag using tfcrop agent from file |
307 336 | # Note : For this test, scan=4 gives identical flags on 32/64 bit machines, |
308 337 | # and one flag difference on a Mac (32) |
309 338 | # Other scans give differences at the 0.005% level. |
310 339 | myinput = "scan='4' mode=tfcrop correlation='ABS_RR' extendflags=False" |
311 - | filename = create_input(myinput) |
312 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True, |
340 + | self.filename = create_input(myinput) |
341 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=True, |
313 342 | flagbackup=False) |
314 343 | res = flagdata(vis=self.vis,mode='summary') |
315 344 | self.assertEqual(res['scan']['1']['flagged'], 568134, 'Whole scan=1 should be flagged') |
316 345 | #self.assertEqual(res['scan']['4']['flagged'], 1201, 'scan=4 should be partially flagged') |
317 346 | self.assertTrue(res['scan']['4']['flagged']>= 1200 and res['scan']['4']['flagged']<= 1204, \ |
318 347 | 'scan=4 should be partially flagged') |
319 348 | |
320 349 | # Unapply only the tfcrop line |
321 350 | flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=1, savepars=False) |
322 351 | result = flagdata(vis=self.vis,mode='summary',scan='4') |
323 352 | self.assertEqual(result['flagged'], 0, 'Expected 0 flags, found %s'%result['flagged']) |
324 353 | self.assertEqual(result['total'], 95256,'Expected total 95256, found %s'%result['total']) |
325 354 | |
326 355 | def test_unapply_tfcrop_and_unset_flagrow(self): |
327 356 | '''flagcmd: Check that FLAG_ROW is unset after un-applying an tfcrop agent''' |
328 357 | # Remove any cmd from table |
329 358 | flagcmd(vis=self.vis, action='clear', clearall=True) |
330 359 | |
331 360 | # Flag using manual agent |
332 361 | myinput = "scan='4'" |
333 - | filename = create_input(myinput) |
334 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False) |
362 + | self.filename = create_input(myinput) |
363 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=False) |
335 364 | |
336 365 | # Check FLAG_ROW is all set to true |
337 366 | mytb = table() |
338 367 | mytb.open(self.vis) |
339 368 | selectedtb = mytb.query('SCAN_NUMBER in [4]') |
340 369 | FLAG_ROW = selectedtb.getcol('FLAG_ROW') |
341 370 | mytb.close() |
342 371 | selectedtb.close() |
343 372 | self.assertEqual(FLAG_ROW.sum(), FLAG_ROW.size) |
344 373 | |
345 374 | # Flag using tfcrop agent from file |
346 375 | myinput = "scan='4' mode=tfcrop correlation='ABS_RR' extendflags=False" |
347 - | filename = create_input(myinput) |
348 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True, |
376 + | self.filename = create_input(myinput) |
377 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=True, |
349 378 | flagbackup=False) |
350 379 | |
351 380 | # Check FLAG_ROW is all set to true |
352 381 | mytb = table() |
353 382 | mytb.open(self.vis) |
354 383 | selectedtb = mytb.query('SCAN_NUMBER in [4]') |
355 384 | FLAG_ROW = selectedtb.getcol('FLAG_ROW') |
356 385 | mytb.close() |
357 386 | selectedtb.close() |
358 387 | self.assertEqual(FLAG_ROW.sum(), FLAG_ROW.size) |
369 398 | selectedtb.close() |
370 399 | self.assertEqual(FLAG_ROW.sum(), 0) |
371 400 | |
372 401 | def test_unapply_rflag_and_unset_flagrow(self): |
373 402 | '''flagcmd: Check that FLAG_ROW is unset after un-applying an rflag agent''' |
374 403 | # Remove any cmd from table |
375 404 | flagcmd(vis=self.vis, action='clear', clearall=True) |
376 405 | |
377 406 | # Flag using manual agent |
378 407 | myinput = "scan='4'" |
379 - | filename = create_input(myinput) |
380 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False) |
408 + | self.filename = create_input(myinput) |
409 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=False) |
381 410 | |
382 411 | # Check FLAG_ROW is all set to true |
383 412 | mytb = table() |
384 413 | mytb.open(self.vis) |
385 414 | selectedtb = mytb.query('SCAN_NUMBER in [4]') |
386 415 | FLAG_ROW = selectedtb.getcol('FLAG_ROW') |
387 416 | mytb.close() |
388 417 | selectedtb.close() |
389 418 | self.assertEqual(FLAG_ROW.sum(), FLAG_ROW.size) |
390 419 | |
391 420 | # Flag using tfcrop agent from file |
392 421 | myinput = "scan='4' mode=rflag " |
393 - | filename = create_input(myinput) |
394 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True, |
422 + | self.filename = create_input(myinput) |
423 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=True, |
395 424 | flagbackup=False) |
396 425 | |
397 426 | # Check FLAG_ROW is all set to true |
398 427 | mytb = table() |
399 428 | mytb.open(self.vis) |
400 429 | selectedtb = mytb.query('SCAN_NUMBER in [4]') |
401 430 | FLAG_ROW = selectedtb.getcol('FLAG_ROW') |
402 431 | mytb.close() |
403 432 | selectedtb.close() |
404 433 | self.assertEqual(FLAG_ROW.sum(), FLAG_ROW.size) |
415 444 | selectedtb.close() |
416 445 | self.assertEqual(FLAG_ROW.sum(), 0) |
417 446 | |
418 447 | def test_unapply_clip_and_unset_flagrow(self): |
419 448 | '''flagcmd: Check that FLAG_ROW is unset after un-applying a clip agent''' |
420 449 | # Remove any cmd from table |
421 450 | flagcmd(vis=self.vis, action='clear', clearall=True) |
422 451 | |
423 452 | # Flag using manual agent |
424 453 | myinput = "scan='4'" |
425 - | filename = create_input(myinput) |
426 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False) |
454 + | self.filename = create_input(myinput) |
455 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=False) |
427 456 | |
428 457 | # Check FLAG_ROW is all set to true |
429 458 | mytb = table() |
430 459 | mytb.open(self.vis) |
431 460 | selectedtb = mytb.query('SCAN_NUMBER in [4]') |
432 461 | FLAG_ROW = selectedtb.getcol('FLAG_ROW') |
433 462 | mytb.close() |
434 463 | selectedtb.close() |
435 464 | self.assertEqual(FLAG_ROW.sum(), FLAG_ROW.size) |
436 465 | |
437 466 | # Flag using clip agent from file |
438 467 | myinput = "scan='4' mode=clip " |
439 - | filename = create_input(myinput) |
440 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True, |
468 + | self.filename = create_input(myinput) |
469 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=True, |
441 470 | flagbackup=False) |
442 471 | |
443 472 | # Check FLAG_ROW is all set to true |
444 473 | mytb = table() |
445 474 | mytb.open(self.vis) |
446 475 | selectedtb = mytb.query('SCAN_NUMBER in [4]') |
447 476 | FLAG_ROW = selectedtb.getcol('FLAG_ROW') |
448 477 | mytb.close() |
449 478 | selectedtb.close() |
450 479 | self.assertEqual(FLAG_ROW.sum(), FLAG_ROW.size) |
462 491 | self.assertEqual(FLAG_ROW.sum(), 0) |
463 492 | |
464 493 | def test_unapply_antint(self): |
465 494 | '''flagcmd: unapply antint agent''' |
466 495 | # Remove any cmd from table |
467 496 | flagcmd(vis=self.vis, action='clear', clearall=True) |
468 497 | |
469 498 | ant_name = 'VA01' |
470 499 | # Flag with antint |
471 500 | in_cmd = "antint_ref_antenna={0} spw='0' minchanfrac=0.8".format(ant_name) |
472 - | filename = create_input(in_cmd) |
473 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', |
501 + | self.filename = create_input(in_cmd) |
502 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', |
474 503 | savepars=True, flagbackup=False) |
475 504 | |
476 505 | res = flagdata(vis=self.vis, mode='summary') |
477 506 | antint_flagged = res['antenna'][ant_name]['flagged'] |
478 507 | |
479 508 | # Flag using manual agent |
480 509 | in_manual = "scan='1'" |
481 - | filename = create_input(in_manual) |
482 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', |
510 + | self.filename = create_input(in_manual) |
511 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', |
483 512 | savepars=True, flagbackup=False) |
484 513 | result = flagdata(vis=self.vis,mode='summary') |
485 514 | man_flagged = result['antenna'][ant_name]['flagged'] |
486 515 | |
487 516 | # Unapply only the antint line |
488 517 | flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=0, |
489 518 | savepars=True) |
490 519 | unapplied_res = flagdata(vis=self.vis, mode='summary') |
491 520 | unapplied_flagged = unapplied_res['antenna'][ant_name]['flagged'] |
492 521 | |
497 526 | self.assertEqual(unapplied_flagged, 0) |
498 527 | self.assertEqual(unapplied_res['antenna'][ant_name]['flagged'], 0) |
499 528 | |
500 529 | def test_uquack(self): |
501 530 | '''flagcmd: unapply quack agent''' |
502 531 | # Remove any cmd from table |
503 532 | flagcmd(vis=self.vis, action='clear', clearall=True) |
504 533 | |
505 534 | # Flag using the quack agent |
506 535 | myinput = "scan='1~3' mode=quack quackinterval=1.0" |
507 - | filename = create_input(myinput) |
508 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True, |
536 + | self.filename = create_input(myinput) |
537 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=True, |
509 538 | flagbackup=False) |
510 539 | result = flagdata(vis=self.vis,mode='summary') |
511 540 | quack_flags = result['scan']['1']['flagged'] |
512 541 | |
513 542 | # Flag using manual agent |
514 543 | myinput = "scan='1'" |
515 - | filename = create_input(myinput) |
516 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True, |
544 + | self.filename = create_input(myinput) |
545 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=True, |
517 546 | flagbackup=False) |
518 547 | result = flagdata(vis=self.vis,mode='summary') |
519 548 | scan1_flags = result['scan']['1']['flagged'] |
520 549 | |
521 550 | # Unapply only the quack line |
522 551 | flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=0, savepars=True) |
523 552 | result = flagdata(vis=self.vis,mode='summary') |
524 553 | manual_flags = result['scan']['1']['flagged'] |
525 554 | |
526 555 | # Only the manual flags should be there |
530 559 | self.assertEqual(result['scan']['3']['flagged'], 0) |
531 560 | self.assertEqual(manual_flags,scan1_flags - quack_flags) |
532 561 | |
533 562 | def test_umanualflag(self): |
534 563 | '''flagcmd: unapply manual agent''' |
535 564 | # Remove any cmd from table |
536 565 | flagcmd(vis=self.vis, action='clear', clearall=True) |
537 566 | |
538 567 | # Flag using manual agent |
539 568 | myinput = "scan='1'" |
540 - | filename = create_input(myinput) |
541 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True, |
569 + | self.filename = create_input(myinput) |
570 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=True, |
542 571 | flagbackup=False) |
543 572 | |
544 573 | # Flag using the quack agent |
545 574 | myinput = "scan='1~3' mode=quack quackinterval=1.0" |
546 - | filename = create_input(myinput) |
547 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True, |
575 + | self.filename = create_input(myinput) |
576 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=True, |
548 577 | flagbackup=False) |
549 578 | |
550 579 | # Unapply the manual line |
551 580 | flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=0, savepars=False) |
552 581 | result = flagdata(vis=self.vis,mode='summary',scan='1,2,3') |
553 582 | |
554 583 | # scan 1 should be fully unflagged |
555 584 | self.assertEqual(result['scan']['1']['flagged'], 0) |
556 585 | self.assertEqual(result['scan']['2']['flagged'], 47628) |
557 586 | self.assertEqual(result['scan']['3']['flagged'], 47628) |
581 610 | self.assertEqual(res['scan']['1']['flagged'], 0, "It should not re-apply tablerows=1") |
582 611 | self.assertEqual(res['scan']['4']['flagged'], 95256, "It should not unapply tablerows=4") |
583 612 | self.assertEqual(res['scan']['7']['flagged'], 190512, "It should not unapply tablerows=7") |
584 613 | self.assertEqual(res['flagged'], 285768) |
585 614 | |
586 615 | |
587 616 | class test_savepars(test_base): |
588 617 | # Action apply, with savepars=True/False |
589 618 | def setUp(self): |
590 619 | self.setUp_ngc5921() |
620 + | self.filename = '' |
621 + | |
622 + | def tearDown(self) -> None: |
623 + | if os.path.exists(self.filename): |
624 + | os.system('rm -rf '+self.filename) |
625 + | |
626 + | |
627 + | def tearDownClass(cls) -> None: |
628 + | shutil.rmtree('ngc5921.ms', ignore_errors=True) |
629 + | shutil.rmtree('ngc5921.ms.flagversions', ignore_errors=True) |
630 + | if os.path.exists('myflags.txt'): |
631 + | os.system('rm -rf myflags.txt') |
632 + | if os.path.exists('newfile.txt'): |
633 + | os.system('rm -rf newfile.txt') |
634 + | if os.path.exists('filename1.txt'): |
635 + | os.system('rm -rf filename1.txt') |
591 636 | |
592 637 | def test_list1(self): |
593 638 | '''flagcmd: list and savepars=True/False''' |
594 639 | # Remove any cmd from table |
595 640 | flagcmd(vis=self.vis, action='clear', clearall=True) |
596 641 | |
597 642 | ########## TEST 1 |
598 643 | # create text file called flagcmd.txt |
599 644 | myinput = "scan='4' mode='clip' correlation='ABS_RR' clipminmax=[0, 4]\n" |
600 - | filename = create_input(myinput) |
645 + | self.filename = create_input(myinput) |
601 646 | filename1 = 'filename1.txt' |
602 - | os.system('cp '+filename+' '+filename1) |
647 + | os.system('cp '+self.filename+' '+filename1) |
603 648 | |
604 649 | # save command to MS |
605 650 | flagcmd(vis=self.vis, action='list', inpmode='list', inpfile=[myinput], savepars=True) |
606 651 | |
607 652 | # list/save to a file |
608 653 | os.system('rm -rf myflags.txt') |
609 654 | flagcmd(vis=self.vis, action='list', outfile='myflags.txt', savepars=True) |
610 655 | |
611 656 | # compare saved file with original input file |
612 657 | self.assertTrue(filecmp.cmp(filename1, 'myflags.txt', 1), 'Files should be equal') |
613 658 | |
614 659 | ########## TEST 2 |
615 660 | # create another input |
616 661 | myinput = "scan='1~3' mode='manual'\n" |
617 - | filename = create_input(myinput) |
662 + | self.filename = create_input(myinput) |
618 663 | |
619 664 | # apply and don't save to MS |
620 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False, |
665 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, action='apply', savepars=False, |
621 666 | flagbackup=False) |
622 667 | |
623 668 | # list and check that parameters were not saved to MS |
624 669 | os.system('rm -rf myflags.txt') |
625 670 | flagcmd(vis=self.vis, action='list', outfile='myflags.txt', savepars=True) |
626 - | self.assertFalse(filecmp.cmp(filename, 'myflags.txt', 1), 'Files should not be equal') |
671 + | self.assertFalse(filecmp.cmp(self.filename, 'myflags.txt', 1), 'Files should not be equal') |
627 672 | |
628 673 | ########### TEST 3 |
629 674 | # apply cmd from TEST 1 and update APPLIED column |
630 675 | flagcmd(vis=self.vis, action='apply', savepars=False, flagbackup=False) |
631 676 | |
632 677 | # scans=1~3 should be fully flagged |
633 678 | res = flagdata(vis=self.vis, mode='summary') |
634 679 | self.assertEqual(res['scan']['1']['flagged'], 568134) |
635 680 | self.assertEqual(res['scan']['1']['total'], 568134) |
636 681 | self.assertEqual(res['scan']['2']['flagged'], 238140) |
637 682 | self.assertEqual(res['scan']['2']['total'], 238140) |
638 683 | self.assertEqual(res['scan']['3']['flagged'], 762048) |
639 684 | self.assertEqual(res['scan']['3']['total'], 762048) |
640 685 | # scan 4 should be partially flagged by clip mode |
641 686 | self.assertEqual(res['scan']['4']['flagged'], 3348, 'Only RR should be flagged') |
642 687 | self.assertEqual(res['scan']['4']['total'], 95256) |
643 688 | |
644 689 | # Only cmd from TEST 1 should be in MS |
645 690 | os.system('rm -rf myflags.txt') |
646 691 | flagcmd(vis=self.vis, action='list', outfile='myflags.txt', useapplied=True, savepars=True) |
647 - | self.assertTrue(filecmp.cmp(filename1, 'myflags.txt', 1), 'Files should be equal') |
692 + | self.assertTrue(filecmp.cmp(filename1, 'myflags.txt', 1), 'Files should be equal') |
648 693 | |
649 694 | def test_overwrite_true(self): |
650 695 | '''flagcmd: Use savepars and overwrite=True''' |
651 696 | |
652 697 | # Create flag commands in file called flagcmd.txt |
653 698 | myinput = "scan='4' mode='clip' correlation='ABS_RR' clipminmax=[0, 4]\n" |
654 - | filename = create_input(myinput) |
699 + | self.filename = create_input(myinput) |
655 700 | # Copy it to a new file |
656 701 | newfile = 'newfile.txt' |
657 702 | os.system('rm -rf '+newfile) |
658 - | os.system('cp '+filename+' '+newfile) |
703 + | os.system('cp '+self.filename+' '+newfile) |
659 704 | |
660 705 | # Create different flag command |
661 706 | myinput = "scan='1'\n" |
662 - | filename = create_input(myinput) |
707 + | self.filename = create_input(myinput) |
663 708 | |
664 709 | # Apply flags from filename and try to save in newfile |
665 710 | # Overwrite parameter should allow this |
666 - | flagcmd(vis=self.vis, action='apply', inpmode='list',inpfile=filename, savepars=True, outfile=newfile, |
711 + | flagcmd(vis=self.vis, action='apply', inpmode='list',inpfile=self.filename, savepars=True, outfile=newfile, |
667 712 | flagbackup=False) |
668 713 | |
669 714 | # newfile should contain what was in filename |
670 - | self.assertTrue(filecmp.cmp(filename, newfile, 1), 'Files should be equal') |
715 + | self.assertTrue(filecmp.cmp(self.filename, newfile, 1), 'Files should be equal') |
671 716 | |
672 717 | def test_overwrite_false(self): |
673 718 | '''flagcmd: Use savepars and overwrite=False''' |
674 719 | |
675 720 | # Create flag commands in file called flagcmd.txt |
676 721 | myinput = "scan='4' mode='clip' correlation='ABS_RR' clipminmax=[0, 4]\n" |
677 - | filename = create_input(myinput) |
722 + | self.filename = create_input(myinput) |
678 723 | # Copy it to a new file |
679 724 | newfile = 'newfile.txt' |
680 725 | os.system('rm -rf '+newfile) |
681 - | os.system('cp '+filename+' '+newfile) |
726 + | os.system('cp '+self.filename+' '+newfile) |
682 727 | |
683 728 | # Create different flag command |
684 729 | myinput = "scan='1'\n" |
685 - | filename = create_input(myinput) |
730 + | self.filename = create_input(myinput) |
686 731 | |
687 732 | passes = False |
688 733 | try: |
689 734 | # Apply flags from filename and try to save in newfile |
690 735 | # Overwrite parameter at False should reject this |
691 - | flagcmd(vis=self.vis, action='apply', inpmode='list',inpfile=filename, savepars=True, outfile=newfile, |
736 + | flagcmd(vis=self.vis, action='apply', inpmode='list',inpfile=self.filename, savepars=True, outfile=newfile, |
692 737 | flagbackup=False, overwrite=False) |
693 738 | # CASA5 gets here, that's OK |
694 739 | passes = True |
695 740 | |
696 741 | # do this additional test for CASA5 to make sure |
697 742 | # newfile should contain what was in filename |
698 - | self.assertFalse(filecmp.cmp(filename, newfile, 1), 'Files should be different') |
743 + | self.assertFalse(filecmp.cmp(self.filename, newfile, 1), 'Files should be different') |
699 744 | except: |
700 745 | # CASA 6 throws an exception |
701 746 | print('Expected error!') |
702 747 | passes = True |
703 748 | |
704 749 | self.assertTrue(passes) |
705 750 | |
706 751 | def test_overwrite_false1(self): |
707 752 | '''flagcmd: Use savepars and overwrite=False''' |
708 753 | |
709 754 | # Remove any cmd from table |
710 755 | flagcmd(vis=self.vis, action='clear', clearall=True) |
711 756 | |
712 757 | # Create flag commands in file called flagcmd.txt |
713 758 | myinput = "scan='4' mode='clip' correlation='ABS_RR' clipminmax=[0, 4]\n" |
714 - | filename = create_input(myinput) |
759 + | self.filename = create_input(myinput) |
715 760 | |
716 761 | newfile = 'myflags.txt' |
717 762 | if os.path.exists("myflags.txt"): |
718 763 | os.system('rm -rf myflags.txt') |
719 764 | |
720 765 | # Apply flags from filename and try to save in newfile |
721 766 | # Overwrite=False shouldn't do anything as newfile doesn't exist |
722 - | flagcmd(vis=self.vis, action='apply', inpmode='list',inpfile=filename, savepars=True, outfile=newfile, |
767 + | flagcmd(vis=self.vis, action='apply', inpmode='list',inpfile=self.filename, savepars=True, outfile=newfile, |
723 768 | flagbackup=False, overwrite=False) |
724 769 | |
725 770 | # newfile should contain what was in filename |
726 - | self.assertTrue(filecmp.cmp(filename, newfile, 1), 'Files should be the same') |
771 + | self.assertTrue(filecmp.cmp(self.filename, newfile, 1), 'Files should be the same') |
727 772 | |
728 773 | |
729 774 | class test_XML(test_base): |
730 775 | |
731 776 | def setUp(self): |
732 777 | self.setUp_evla() |
733 - | |
778 + | self.filename = '' |
779 + | self.filename1 = '' |
780 + | |
734 781 | # During implementation of CAS-13049, test_xml2 would fail without a tearDown |
735 782 | def tearDown(self): |
736 783 | os.system('rm -rf tosr0001_scan3*.ms*') |
784 + | if os.path.exists(self.filename): |
785 + | os.system('rm -rf '+self.filename) |
786 + | if os.path.exists(self.filename1): |
787 + | os.system('rm -rf '+self.filename1) |
737 788 | |
738 - | |
739 789 | def test_xml1(self): |
740 790 | '''flagcmd: list xml file and save in outfile''' |
741 791 | |
742 792 | # The MS only contains clip and shadow commands |
743 793 | # The XML contain the online flags |
744 - | flagcmd(vis=self.vis, action='list', inpmode='xml', savepars=True, outfile='origxml.txt') |
794 + | self.filename= 'origxml.txt' |
795 + | flagcmd(vis=self.vis, action='list', inpmode='xml', savepars=True, outfile=self.filename) |
745 796 | |
746 797 | # Now save the online flags to the FLAG_CMD without applying |
747 798 | flagcmd(vis=self.vis, action='list', inpmode='xml', savepars=True) |
748 799 | |
749 800 | # Now apply them by selecting the reasons and save in another file |
750 801 | # 507 cmds crash on my computer. |
751 802 | reasons = ['ANTENNA_NOT_ON_SOURCE','FOCUS_ERROR','SUBREFLECTOR_ERROR'] |
752 - | flagcmd(vis=self.vis, action='apply', reason=reasons, savepars=True, outfile='myxml.txt', |
803 + | self.filename1= 'myxml.txt' |
804 + | flagcmd(vis=self.vis, action='apply', reason=reasons, savepars=True, outfile=self.filename1, |
753 805 | flagbackup=False) |
754 806 | |
755 807 | # Compare with original XML |
756 - | self.assertTrue(filecmp.cmp('origxml.txt', 'myxml.txt',1), 'Files should be equal') |
808 + | self.assertTrue(filecmp.cmp(self.filename, self.filename1,1), 'Files should be equal') |
757 809 | |
758 810 | # Check that APPLIED column has been updated to TRUE |
759 811 | |
760 - | |
761 812 | def test_xml2(self): |
762 813 | '''flagcmd: list xml file and save in outfile''' |
763 814 | |
764 815 | # The MS only contains clip and shadow commands |
765 816 | |
766 817 | # CASA6 needs this |
767 818 | os.system('rsync -a '+os.path.join(datapath,self.vis)+' .') |
768 819 | # Apply the shadow command |
769 820 | flagcmd(vis=self.vis, action='apply', reason='SHADOW', flagbackup=False) |
770 821 | res = flagdata(vis=self.vis, mode='summary') |
777 828 | flagcmd(vis=self.vis, inpmode='list', inpfile=["correlation='XX,RR,RL'"], action='list', |
778 829 | savepars=True) |
779 830 | |
780 831 | flagcmd(vis=self.vis, action='apply', flagbackup=False) |
781 832 | |
782 833 | res = flagdata(vis=self.vis, mode='summary') |
783 834 | |
784 835 | self.assertEqual(res['flagged'], 208000+208000, 'Should only flag RR and RL and not fail') |
785 836 | |
786 837 | class test_shadow(test_base): |
838 + | filename = '' |
787 839 | def setUp(self): |
788 840 | self.setUp_shadowdata() |
789 841 | |
842 + | def tearDown(self) -> None: |
843 + | shutil.rmtree(self.vis, ignore_errors=True) |
844 + | if os.path.exists(self.filename): |
845 + | os.system('rm -rf '+self.filename) |
846 + | |
790 847 | def test_CAS2399(self): |
791 848 | '''flagcmd: shadow by antennas not present in MS''' |
792 849 | |
793 850 | # Create antennafile in disk |
794 851 | myinput = 'name=VLA01\n'+\ |
795 852 | 'diameter=25.0\n'+\ |
796 853 | 'position=[-1601144.96146691, -5041998.01971858, 3554864.76811967]\n'+\ |
797 854 | 'name=VLA02\n'+\ |
798 855 | 'diameter=25.0\n'+\ |
799 856 | 'position=[-1601105.7664601889, -5042022.3917835914, 3554847.245159178]\n'+\ |
800 857 | 'name=VLA09\n'+\ |
801 858 | 'diameter=25.0\n'+\ |
802 859 | 'position=[-1601197.2182404203, -5041974.3604805721, 3554875.1995636248]\n'+\ |
803 860 | 'name=VLA10\n'+\ |
804 861 | 'diameter=25.0\n'+\ |
805 862 | 'position=[-1601227.3367843349,-5041975.7011900628,3554859.1642644769]\n' |
806 863 | |
807 864 | # antfile = 'myants.txt' |
808 865 | # if os.path.exists(antfile): |
809 866 | # os.system('rm -rf myants.txt') |
810 867 | |
811 - | filename = create_input(myinput) |
868 + | self.filename = create_input(myinput) |
812 869 | |
813 870 | # Create command line |
814 871 | myinput = ["mode='shadow' tolerance=0.0 addantenna='flagcmd.txt'"] |
815 872 | # filename = 'cmdfile.txt' |
816 873 | # if os.path.exists(filename): |
817 874 | # os.system('rm -rf cmdfile.txt') |
818 875 | |
819 876 | # create_input(myinput, filename) |
820 877 | |
821 878 | # Flag |
829 886 | self.assertEqual(res['antenna']['VLA4']['flagged'], 1320) |
830 887 | self.assertEqual(res['antenna']['VLA5']['flagged'], 1104) |
831 888 | |
832 889 | |
833 890 | class test_antint(test_base): |
834 891 | ''' For antint mode with inputs from files''' |
835 892 | |
836 893 | def setUp(self): |
837 894 | self.setUp_evla() |
838 895 | |
896 + | |
897 + | def tearDownClass(cls) -> None: |
898 + | shutil.rmtree('tosr0001_scan3_noonline.ms', ignore_errors=True) |
899 + | |
839 900 | def test_antint_defaults(self): |
840 901 | '''flagcmd:: Test of antint with defaults''' |
841 902 | |
842 903 | ant_name = 'ea01' |
843 904 | spw_spec = '0' |
844 905 | flagdata(vis=self.vis, mode='unflag'); |
845 906 | flagcmd(vis=self.vis, inpmode='list', |
846 907 | inpfile=["mode=antint antint_ref_antenna='{0}' spw='{1}'".format(ant_name, |
847 908 | spw_spec), |
848 909 | "mode=summary spw='{0}'".format(spw_spec)], |
877 938 | self.assertEqual(res_str['antenna'][antenna_spec]['flagged'], 32000) |
878 939 | |
879 940 | |
880 941 | # Test rflag inputs with filenames, as well as inline thresholds. |
881 942 | # rflag does not generate output thresholds for action='apply'. |
882 943 | class test_rflag(test_base): |
883 944 | |
884 945 | def setUp(self): |
885 946 | self.setUp_data4rflag() |
886 947 | |
948 + | def tearDown(self) -> None: |
949 + | if os.path.exists('rflag_output_thresholds_freqdev0.txt'): |
950 + | os.system('rm -rf rflag_output_thresholds*') |
951 + | if os.path.exists('fdevfile.txt'): |
952 + | os.system('rm -rf *devfile.txt') |
953 + | if os.path.exists('outcmd.txt'): |
954 + | os.system('rm -rf outcmd.txt') |
955 + | |
956 + | |
957 + | def tearDownClass(cls) -> None: |
958 + | os.system('rm -rf Four_ants_3C286.ms*') |
959 + | |
887 960 | def test_rflaginputs(self): |
888 961 | """flagcmd:: Test of rflag threshold-inputs of both types (file and inline) |
889 962 | """ |
890 963 | |
891 964 | # (1) and (2) are the same as test_flagdata[test_rflag3]. |
892 965 | # -- (3),(4),(5) should produce the same answers as (1) and (2) |
893 966 | # |
894 967 | # (1) Test input/output files, through the task, mode='rflag' |
895 968 | #flagdata(vis=self.vis, mode='rflag', spw='9,10', timedev='tdevfile.txt', \ |
896 969 | # freqdev='fdevfile.txt', action='calculate'); |
988 1061 | #print("\n---------------------- Finished flagdata test : auto : ", res7['flagged']) |
989 1062 | |
990 1063 | self.assertEqual(res6['flagged'], 42728.0) |
991 1064 | |
992 1065 | |
993 1066 | class test_actions(test_base): |
994 1067 | |
995 1068 | def setUp(self): |
996 1069 | self.setUp_data4rflag() |
997 1070 | |
998 - | def tearDown(self): |
999 - | shutil.rmtree(self.vis, ignore_errors=True) |
1000 - | os.system('rm -rf flagcmd.txt') |
1001 - | # if os.path.exists('fourplot.png'): |
1002 - | # os.remove('fourplot.png') |
1003 - | |
1071 + | def tearDown(self) -> None: |
1072 + | os.system('rm -rf list*txt *plot*png reasonfile*txt') |
1073 + | if os.path.exists('flagcmd.txt'): |
1074 + | os.system('rm -rf flagcmd.txt') |
1075 + | if os.path.exists('cas9366.flags.txt'): |
1076 + | os.system('rm -rf cas9366.flags.txt') |
1077 + | |
1078 + | |
1079 + | def tearDownClass(cls) -> None: |
1080 + | os.system('rm -rf ngc5921.ms*') |
1081 + | os.system('rm -rf Four_ants_3C286.ms*') |
1082 + | |
1004 1083 | def test_action_plot_table(self): |
1005 1084 | '''flagcmd: Test action=plot, nothing plotted''' |
1006 1085 | outplot = 'noplot.png' |
1007 1086 | flagcmd(vis=self.vis, inpmode='list', |
1008 1087 | inpfile=["intent='CAL*POINT*' field=''","scan='5'"], |
1009 1088 | action='list', savepars=True) |
1010 1089 | |
1011 1090 | flagcmd(vis=self.vis, inpmode='table', useapplied=True, action='plot', |
1012 1091 | plotfile=outplot) |
1013 1092 | |
1065 1144 | flagcmd(vis=self.vis) |
1066 1145 | |
1067 1146 | res=flagdata(vis=self.vis, mode='summary') |
1068 1147 | self.assertEqual(res['flagged'],1099776) |
1069 1148 | |
1070 1149 | def test_action_list2(self): |
1071 1150 | '''flagcmd: action=list with inpmode from a file''' |
1072 1151 | flagcmd(vis=self.vis, action='clear', clearall=True) |
1073 1152 | cmd = "spw='5~7'\n"+\ |
1074 1153 | "spw='1'" |
1075 - | filename = create_input(cmd) |
1076 - | flagcmd(vis=self.vis, action='list', inpmode='list', inpfile=filename, savepars=True) |
1154 + | self.filename = create_input(cmd) |
1155 + | flagcmd(vis=self.vis, action='list', inpmode='list', inpfile=self.filename, savepars=True) |
1077 1156 | |
1078 1157 | # Apply the flags |
1079 1158 | flagcmd(vis=self.vis) |
1080 1159 | |
1081 1160 | res=flagdata(vis=self.vis, mode='summary') |
1082 1161 | self.assertEqual(res['flagged'],1099776) |
1083 1162 | |
1084 1163 | def test_CAS4819(self): |
1085 1164 | '''flagcmd: CAS-4819, Flag commands from three files''' |
1086 1165 | self.setUp_ngc5921() |
1173 1252 | class test_cmdbandpass(test_base): |
1174 1253 | """Flagcmd:: Test flagging task with Bpass-based CalTable """ |
1175 1254 | |
1176 1255 | def setUp(self): |
1177 1256 | self.setUp_bpass_case() |
1178 1257 | |
1179 1258 | def tearDown(self): |
1180 1259 | shutil.rmtree(self.vis, ignore_errors=True) |
1181 1260 | os.system('rm -rf flagcmd.txt') |
1182 1261 | |
1262 + | |
1263 + | def tearDownClass(cls) -> None: |
1264 + | os.system('rm -rf cal.fewscans*') |
1265 + | os.system('rm -rf Four_ants_3C286.ms*') |
1266 + | if os.path.exists('calflags.txt'): |
1267 + | os.system('rm -rf calflags.txt') |
1268 + | |
1183 1269 | def test_unsupported_mode_in_list(self): |
1184 1270 | '''Flagcmd: elevation and shadow are not supported in cal tables''' |
1185 1271 | res = flagcmd(vis=self.vis, inpmode='list', inpfile=["mode='elevation'", |
1186 1272 | "spw='1'"]) |
1187 1273 | |
1188 1274 | res = flagdata(vis=self.vis, mode='summary') |
1189 1275 | self.assertEqual(res['spw']['1']['flagged'], 83200) |
1190 1276 | |
1191 1277 | def test_default_cparam(self): |
1192 1278 | '''Flagcmd: flag CPARAM data column''' |
1204 1290 | |
1205 1291 | self.assertEqual(summary['field']['3C286_A']['flagged'], 499200.0) |
1206 1292 | self.assertEqual(summary['field']['3C286_B']['flagged'], 0) |
1207 1293 | self.assertEqual(summary['field']['3C286_C']['flagged'], 0) |
1208 1294 | self.assertEqual(summary['field']['3C286_D']['flagged'], 0) |
1209 1295 | |
1210 1296 | def test_list_field_Selection_for_bpass(self): |
1211 1297 | """Flagcmd:: Manually flag a bpass-based CalTable using file in list mode """ |
1212 1298 | |
1213 1299 | myinput = "field='3C286_A'" |
1214 - | filename = create_input(myinput) |
1300 + | self.filename = create_input(myinput) |
1215 1301 | |
1216 - | flagcmd(vis=self.vis, inpmode='list', inpfile=filename, flagbackup=False) |
1302 + | flagcmd(vis=self.vis, inpmode='list', inpfile=self.filename, flagbackup=False) |
1217 1303 | summary=flagdata(vis=self.vis, mode='summary') |
1218 1304 | self.assertEqual(summary['field']['3C286_A']['flagged'], 499200.0) |
1219 1305 | self.assertEqual(summary['field']['3C286_B']['flagged'], 0) |
1220 1306 | self.assertEqual(summary['field']['3C286_C']['flagged'], 0) |
1221 1307 | self.assertEqual(summary['field']['3C286_D']['flagged'], 0) |
1222 1308 | |
1223 1309 | def test_MS_flagcmds(self): |
1224 1310 | """Flagcmd:: Save flags to MS and apply to cal table""" |
1225 1311 | self.setUp_data4rflag() |
1226 1312 | msfile = self.vis |
1330 1416 | def test_cal_observation(self): |
1331 1417 | '''Flagcmd: flag an observation from an old cal table format''' |
1332 1418 | # Note: this cal table does not have an observation column. |
1333 1419 | # The column and sub-table should be added and the flagging |
1334 1420 | # should happen after this. |
1335 1421 | flagcmd(vis=self.vis, inpmode='list', inpfile=["observation='0'"], flagbackup=False) |
1336 1422 | res=flagdata(vis=self.vis, mode='summary') |
1337 1423 | self.assertEqual(res['flagged'],1248000) |
1338 1424 | self.assertEqual(res['total'],1248000) |
1339 1425 | |
1340 - | |
1341 - | # Dummy class which cleans up created files |
1342 - | class cleanup(test_base): |
1343 - | |
1344 - | def tearDown(self): |
1345 - | os.system('rm -rf multiobs.ms*') |
1346 - | os.system('rm -rf uid___A002_X30a93d_X43e_small.ms*') |
1347 - | os.system('rm -rf ngc5921.ms*') |
1348 - | os.system('rm -rf Four_ants*.ms*') |
1349 - | os.system('rm -rf shadowtest*.ms*') |
1350 - | os.system('rm -rf tosr0001_scan3*.ms*') |
1351 - | os.system('rm -rf cal.fewscans.bpass*') |
1352 - | |
1353 - | def test1(self): |
1354 - | '''flagcmd: Cleanup''' |
1355 - | pass |
1356 - | |
1357 1426 | if __name__ == '__main__': |
1358 1427 | unittest.main() |