Commits

Federico Montesino Pouzols authored e77d26e8c52 
more data cleanup, some more asserts, re CAS-11397
6651bbf536b
master

gcwrap/python/scripts/tests/test_flagdata.py

Modified
2026 2026 ​
2027 2027 def test_timeavg1(self):
2028 2028 '''flagdata: clip with time average and compare with mstransform'''
2029 2029 # Create an output with 4 rows
2030 2030 from tasks import split
2031 2031 split(vis=self.vis,outputvis='timeavg.ms',datacolumn='data',spw='9',scan='30',antenna='1&2',
2032 2032 timerange='2010/10/16/14:45:08.50~2010/10/16/14:45:11.50')
2033 2033
2034 2034 # STEP 1
2035 2035 # Create time averaged output in mstransform
2036 - mstransform('timeavg.ms',outputvis='step1_timeavg.ms',datacolumn='data',timeaverage=True,timebin='2s')
2036 + mstransform('timeavg.ms',outputvis='test_residual_step1_timeavg.ms',
2037 + datacolumn='data',timeaverage=True,timebin='2s')
2037 2038
2038 2039 # clip it
2039 - flagdata('step1_timeavg.ms',flagbackup=False, mode='clip',clipminmax=[0.0,0.08])
2040 - res1 = flagdata(vis='step1_timeavg.ms', mode='summary', spwchan=True)
2040 + flagdata('test_residual_step1_timeavg.ms',flagbackup=False, mode='clip',
2041 + clipminmax=[0.0,0.08])
2042 + res1 = flagdata(vis='test_residual_step1_timeavg.ms', mode='summary', spwchan=True)
2041 2043 ​
2042 2044 # STEP 2
2043 2045 # Clip with time averaging.
2044 2046 flagdata('timeavg.ms', flagbackup=False, mode='unflag')
2045 2047 flagdata(vis='timeavg.ms', flagbackup=False, mode='clip', datacolumn='DATA',
2046 2048 timeavg=True, timebin='2s', clipminmax=[0.0,0.08])
2047 2049
2048 2050 # Do another time average in mstransform to have the corrected averaged visibilities
2049 - mstransform('timeavg.ms',outputvis='step2_timeavg.ms',datacolumn='data',timeaverage=True,timebin='2s')
2051 + mstransform('timeavg.ms',outputvis='test_residual_step2_timeavg.ms',
2052 + datacolumn='data',timeaverage=True,timebin='2s')
2050 2053
2051 - res2 = flagdata(vis='step2_timeavg.ms', mode='summary', spwchan=True)
2054 + res2 = flagdata(vis='test_residual_step2_timeavg.ms', mode='summary', spwchan=True)
2052 2055 ​
2053 2056 self.assertEqual(res1['flagged'], res2['flagged'])
2054 2057 self.assertEqual(res2['spw:channel']['0:21']['flagged'], 1)
2055 2058 self.assertEqual(res2['spw:channel']['0:51']['flagged'], 8)
2056 2059 self.assertEqual(res2['spw:channel']['0:60']['flagged'], 7)
2057 2060 ​
2058 2061 def test_timeavg2(self):
2059 2062 '''flagdata: clip with time averaging in spw 9'''
2060 2063
2061 2064 flagdata(vis=self.vis, flagbackup=False, mode='clip', datacolumn='DATA', spw='9',
3344 3347 self.assertEqual(res[1]['timerange'], '00:00~00:01,00:02~00:03')
3345 3348 self.assertEqual(res[2]['mode'], 'manual')
3346 3349 self.assertEqual(res[2]['timerange'], '00:03~00:02')
3347 3350 self.assertEqual(res[3]['mode'], 'manual')
3348 3351 self.assertEqual(res[3]['timerange'], '00:04~00:05')
3349 3352 self.assertEqual(res[4]['mode'], 'summary2')
3350 3353 self.assertEqual(res[5]['mode'], 'manual')
3351 3354 self.assertEqual(res[5]['timerange'], '00:04~00:05,00:06~00:07')
3352 3355 self.assertEqual(res[6]['mode'], 'summary3')
3353 3356 ​
3354 -# Cleanup class
3355 -class cleanup(test_base):
3356 -
3357 - def tearDown(self):
3358 - os.system('rm -rf ngc5921.*ms* testwma*ms*')
3359 - os.system('rm -rf flagdatatest.*ms*')
3360 - os.system('rm -rf missing-baseline.*ms*')
3361 - os.system('rm -rf multiobs.*ms*')
3362 - os.system('rm -rf uid___A002_X30a93d_X43e_small.*ms*')
3363 - os.system('rm -rf Four_ants_3C286.*ms*')
3364 - os.system('rm -rf shadow*.*ms*')
3365 - os.system('rm -rf testmwa.*ms*')
3366 - os.system('rm -rf cal.fewscans.bpass*')
3367 - os.system('rm -rf X7ef.tsys* ap314.gcal*')
3368 - os.system('rm -rf list*txt*')
3369 - os.system('rm -rf fourrows*')
3370 - os.system('rm -rf SDFloatColumn*')
3371 - os.system('rm -rf *weight*ms*')
3372 - os.system('rm -rf uid___A002_X72c4aa_X8f5_scan21_spw18*')
3373 -​
3374 - def test_runTest(self):
3375 - '''flagdata: Cleanup'''
3376 - pass
3377 -​
3378 3357 ​
3379 3358 class test_preaveraging(test_base):
3380 3359 """Test channel/time pre-averaging for visibility-based flagging"""
3381 3360
3382 3361 def setUp(self):
3383 3362 self.setUp_data4preaveraging()
3384 3363
3385 3364 def tearDown(self):
3386 3365 os.system('rm -rf test_preaveraging.ms')
3387 3366 os.system('rm -rf test_clip_timeavg*')
3537 3516 ​
3538 3517 # STEP 2: Flagging with rflag using time average, then time average with mstransform
3539 3518 flagdata(vis=self.vis, flagbackup=False, mode='rflag', datacolumn='DATA',
3540 3519 timeavg=True, timebin='2s', channelavg=True, chanbin=2, extendflags=False)
3541 3520 mstransform(vis=self.vis, outputvis='test_rflag_time_chanavg_step2.ms',datacolumn='data',
3542 3521 timeaverage=True,timebin='2s',chanaverage=True,chanbin=2)
3543 3522 res2 = flagdata(vis='test_rflag_time_chanavg_step2.ms', mode='summary', spwchan=True)
3544 3523 ​
3545 3524 # Check results
3546 3525 self.assertEqual(res1['flagged'], 20)
3547 - self.assertEqual(res2['flagged'], 62)
3526 + for corr in ['RL', 'LL', 'LR', 'RR']:
3527 + self.assertGreaterEqual(res2['correlation'][corr], res1['correlation'][corr])
3548 3528
3549 3529 def test_tfcrop_timeavg(self):
3550 3530 '''flagdata: tfcrop with time average and compare vs mstransform'''
3551 3531
3552 3532 # Unflag the original input data
3553 3533 flagdata(self.vis, flagbackup=False, mode='unflag')
3554 3534
3555 3535 # STEP 1: Time average with mstransform, then flagging with normal tfcrop
3556 3536 mstransform(vis=self.vis,outputvis='test_tfcrop_timeavg_step1.ms',datacolumn='data',
3557 3537 timeaverage=True,timebin='2s')
3558 3538 flagdata(vis='test_tfcrop_timeavg_step1.ms',flagbackup=False, mode='tfcrop',
3559 3539 extendflags=False)
3560 3540 res1 = flagdata(vis='test_tfcrop_timeavg_step1.ms', mode='summary', spwchan=True)
3561 3541
3562 3542 # Unflag the original input data
3563 3543 flagdata(self.vis, flagbackup=False, mode='unflag')
3564 3544 ​
3565 - # STEP 2: Flagging with clip using time average, then time average with mstransform
3545 + # STEP 2: Flagging with tfcrop using time average, then time average with mstransform
3566 3546 flagdata(vis=self.vis, flagbackup=False, mode='tfcrop', datacolumn='DATA',
3567 3547 timeavg=True, timebin='2s', extendflags=False)
3568 3548 mstransform(vis=self.vis,outputvis='test_tfcrop_timeavg_step2.ms',datacolumn='data',
3569 3549 timeaverage=True,timebin='2s')
3570 3550 res2 = flagdata(vis='test_tfcrop_timeavg_step2.ms', mode='summary', spwchan=True)
3571 3551 ​
3572 3552 # Check results
3573 - self.assertEqual(res1['flagged'], 0)
3553 + self.assertEqual(res1['flagged'], 36)
3574 3554 self.assertEqual(res2['flagged'], 0)
3575 3555
3576 3556 def test_tfcrop_chanavg(self):
3577 3557 '''flagdata: tfcrop with chan average and compare vs mstransform'''
3578 3558
3579 3559 # Unflag the original input data
3580 3560 flagdata(self.vis, flagbackup=False, mode='unflag')
3581 3561
3582 3562 # STEP 1: Chan average with mstransform, then flagging with normal tfcrop
3583 3563 mstransform(vis=self.vis,outputvis='test_tfcrop_chanavg_step1.ms',datacolumn='data',
3661 3641 res2 = flagdata(vis=flagged2, spw=spws, action='calculate', mode='rflag',
3662 3642 datacolumn='residual', extendflags=False)
3663 3643 ​
3664 3644 # STEP 3: rflag-calculate with timeavg on residual
3665 3645 res3 = flagdata(vis=self.vis, spw=spws, action='calculate', mode='rflag',
3666 3646 datacolumn='residual', timeavg=True, timebin=timebin,
3667 3647 extendflags=False)
3668 3648 ​
3669 3649 def check_reports_timeavg(report1, report2, report3):
3670 3650 import numpy as np
3671 - print "report 1", report1
3672 - print "report 2", report2
3673 - print "report 3", report3
3674 3651 self.assertEqual(report2['type'], 'rflag')
3675 3652 self.assertEqual(report3['type'], report2['type'])
3676 3653 freq_tol = 1e-1
3677 3654 self.assertTrue(np.allclose(report1['freqdev'], report3['freqdev'],
3678 3655 rtol=freq_tol))
3679 3656 self.assertTrue(np.allclose(report2['freqdev'], report3['freqdev'],
3680 3657 rtol=freq_tol))
3681 3658 # divide 3rd column (thresholds). Matrices have rows like: field, spw, threshold.
3682 3659 report1['timedev'][:,2] = report1['timedev'][:,2] / np.sqrt(8)
3683 3660 time_div_tol = 3.3e-1
3797 3774 # Run setjy to create a normal MODEl column (SOURCE_MODEL)
3798 3775 setjy(vis=self.vis, field='1331+305*',modimage='',standard='Perley-Taylor 99',
3799 3776 scalebychan=False, usescratch=True)
3800 3777
3801 3778 flagdata(vis=self.vis,mode='clip',datacolumn='RESIDUAL_DATA',clipminmax=[2.3,3.1],clipoutside=False)
3802 3779 res = flagdata(vis=self.vis, mode='summary')['flagged']
3803 3780
3804 3781 self.assertEqual(res_virtual, res, 'Flagging using virtual MODEL column differs from normal MODEL column')
3805 3782 ​
3806 3783 ​
3784 +# Cleanup class
3785 +class cleanup(test_base):
3786 +​
3787 + def tearDown(self):
3788 + os.system('rm -rf ngc5921.*ms* testwma*ms*')
3789 + os.system('rm -rf flagdatatest.*ms*')
3790 + os.system('rm -rf missing-baseline.*ms*')
3791 + os.system('rm -rf multiobs.*ms*')
3792 + os.system('rm -rf uid___A002_X30a93d_X43e_small.*ms*')
3793 + os.system('rm -rf Four_ants_3C286*.ms')
3794 + os.system('rm -rf shadow*.*ms*')
3795 + os.system('rm -rf testmwa.*ms*')
3796 + os.system('rm -rf cal.fewscans.bpass*')
3797 + os.system('rm -rf X7ef.tsys* ap314.gcal*')
3798 + os.system('rm -rf list*txt*')
3799 + os.system('rm -rf fourrows*')
3800 + os.system('rm -rf SDFloatColumn*')
3801 + os.system('rm -rf *weight*ms*')
3802 + os.system('rm -rf uid___A002_X72c4aa_X8f5_scan21_spw18*')
3803 + os.system('rm -rf test_residual_step*.ms')
3804 +​
3805 + def test_runTest(self):
3806 + '''flagdata: Cleanup'''
3807 + pass
3808 +​
3809 +​
3807 3810 def suite():
3808 3811 return [test_antint,
3809 3812 test_rflag,
3810 3813 test_tfcrop,
3811 3814 test_shadow,
3812 3815 test_selections,
3813 3816 test_selections2,
3814 3817 test_alma,
3815 3818 test_statistics_queries,
3816 3819 test_msselection,

Everything looks good. We'll let you know here if there's anything you should know about.

Add shortcut