casa / casa5 / 4cf5bea6b4d

Commits

Federico Montesino Pouzols authored 4cf5bea6b4d 17 May 2018

split test_preaveraging_rflag_residual, re CAS-11397

gcwrap/python/scripts/tests/test_flagdata.py

Modified

-109
+            os.system('cp -RH '+datapath + self.vis +' '+ self.vis)
-110
-111
+        os.system('rm -rf ' + self.vis + '.flagversions')
-112
+#        self.unflag_ms()
-113
+        flagdata(vis=self.vis, mode='unflag', flagbackup=False)
-114
+        default(flagdata)
-115
-116
+    def setUp_data4tfcrop(self):
-117
+        '''EVLA MS, 4 ants, scan=30,31 spw=0~15, 64 chans, RR,RL,LR,LL'''
-118
+        self.vis = "Four_ants_3C286.ms"
--
-119
+        if os.path.exists(self.vis):
-120
+            print "The MS is already around, just unflag"
-121
+        else:
-122
+            print "Moving data..."
-123
+            os.system('cp -RH '+datapath + self.vis +' '+ self.vis)
-124
-125
+        os.system('rm -rf ' + self.vis + '.flagversions')
-+
-127
+        self.unflag_ms()
-128
+        default(flagdata)
-129
-130
+    def setUp_shadowdata1(self):
-131
+        '''ALMA ACA observation with one field in APP ref frame'''
-132
+        self.vis = "shadowAPP.ms"
-133
-134
+        if os.path.exists(self.vis):
-135
+            print "The MS is already around, just unflag"
-136
+        else:
-315
-316
+        # Extract only the type 'summary' reports
-317
+        nreps = report_list.keys()
-318
+        for rep in range(len(nreps)):
-319
+            repname = 'report'+str(rep);
-320
+            if(report_list[repname]['type']=='summary'):
-321
+                  summary_list.append(report_list[repname]);
-322
-323
+        return summary_list
-324
-+
-326
+class test_tfcrop(test_base):
-327
+    """flagdata:: Test of mode = 'tfcrop'"""
-328
-329
+    def setUp(self):
-330
+        self.setUp_data4tfcrop()
-331
-332
+    def test_tfcrop1(self):
-333
+        '''flagdata:: Test1 of mode = tfcrop'''
-334
+        flagdata(vis=self.vis, mode='tfcrop', correlation='ABS_RR',ntime=51.0,spw='9',
-335
+                 savepars=False, extendflags=False)
-3361
-3362
+    def test_runTest(self):
-3363
+        '''flagdata: Cleanup'''
-3364
+        pass
-3365
-3366
-3367
+class test_preaveraging(test_base):
-3368
+    """Test channel/time pre-averaging for visibility-based flagging"""
-3369
-3370
+    def setUp(self):
--
+        self.setUp_data4tfcrop()
--
+        mstransform(vis=self.vis,outputvis='test_preaveraging.ms',datacolumn='data',spw='9',scan='30',
--
+                    antenna='1&2',timerange='2010/10/16/14:45:08.50~2010/10/16/14:45:11.50')
--
+        self.vis = 'test_preaveraging.ms'
-+
+        self.setUp_data4preaveraging()
-3372
-3373
+    def tearDown(self):
-3374
+        os.system('rm -rf test_preaveraging.ms')
-3375
+        os.system('rm -rf test_clip_timeavg*')
-3376
+        os.system('rm -rf test_clip_chanavg*')
-3377
+        os.system('rm -rf test_clip_time_chanavg*')
-3378
+        os.system('rm -rf test_rflag_timeavg*')
-3379
+        os.system('rm -rf test_rflag_chanavg*')
-3380
+        os.system('rm -rf test_rflag_time_chanavg*')
-3381
+        os.system('rm -rf test_tfcrop_timeavg*')
-3475
+        # STEP 2: Flagging with rflag using time average, then time average with mstransform
-3476
+        flagdata(vis=self.vis, flagbackup=False, mode='rflag', datacolumn='DATA',
-3477
+                 timeavg=True, timebin='2s', extendflags=False)
-3478
+        mstransform(vis=self.vis,outputvis='test_rflag_timeavg_step2.ms',datacolumn='data',
-3479
+                    timeaverage=True,timebin='2s')
-3480
+        res2 = flagdata(vis='test_rflag_timeavg_step2.ms', mode='summary', spwchan=True)
-3481
-3482
+        # Compare results
-3483
+        self.assertEqual(res2['flagged'], 20)
-3484
--
+    def test_rflag_timeavg_on_residual(self):
--
+        '''flagdata: rflag with timeavg on residual (corrected - model), and compare
--
+        vs mstransform + rflag without timeavg'''
--
--
+        # One that has corrected
--
+        vis = "Four_ants_3C286.ms"
--
+        # Initial integration time is 1s
--
+        timebin = '8s'
--
--
+        # Unflag the original input data
--
+        flagdata(vis, flagbackup=False, mode='unflag')
--
--
+        # using action calculate, which is faster (reduced I/O) and enough to test thresholds
--
+        # using only these spws for speed
--
+        spws = '0,1,2'
--
--
+        # STEP 1: rflag-calculate with original time
--
+        res1 = flagdata(vis=vis, spw=spws, action='calculate', mode='rflag',
--
+                        datacolumn='residual', extendflags=False)
--
--
+        # STEP 2: timeavg with mstransform, then rflag-calculate on residual
--
+        flagged1 = 'test_rflag_timeavg_residual_step2.ms'
--
+        mstransform(vis=vis, spw=spws, outputvis=flagged1,
--
+                    datacolumn='data', timeaverage=True, timebin=timebin)
--
+        res2 = flagdata(vis=vis, spw=spws, action='calculate', mode='rflag',
--
+                        datacolumn='residual', extendflags=False)
--
--
+        # STEP 3: rflag-calculate with timeavg on residual
--
+        res3 = flagdata(vis=vis, spw=spws, action='calculate', mode='rflag',
--
+                        datacolumn='residual', timeavg=True, timebin=timebin,
--
+                        extendflags=False)
--
--
+        def check_reports_timeavg(report1, report2, report3):
--
+            import numpy as np
--
+            report1 = res1['report0']
--
+            report2 = res2['report0']
--
+            report3 = res3['report0']
--
+            self.assertEqual(report2['type'], 'rflag')
--
+            self.assertEqual(report3['type'], report2['type'])
--
+            freq_tol = 1e-1
--
+            self.assertTrue(np.allclose(report1['freqdev'], report2['freqdev'],
--
+                                        rtol=freq_tol))
--
+            self.assertTrue(np.allclose(report2['freqdev'], report3['freqdev'],
--
+                                        rtol=freq_tol))
--
+            time_tol = 1e-1
--
+            self.assertTrue(np.allclose(report1['timedev'], report2['timedev'],
--
+                                        rtol=time_tol))
--
+            # divide 3rd column (thresholds). Matrices have rows like: field, spw, threshold.
--
+            time_div_tol = 2.5e-1
--
+            report1['timedev'][:,2] = report1['timedev'][:,2] / np.sqrt(8)
--
+            self.assertTrue(np.allclose(report1['timedev'],
--
+                                        report3['timedev'], rtol=time_div_tol))
--
--
+        self.assertEqual(res1['type'], 'list')
--
+        self.assertEqual(res1['type'], res2['type'])
--
+        self.assertEqual(res1['type'], res3['type'])
--
+        check_reports_timeavg(res1['report0'], res2['report0'], res3['report0'])
--
-3485
+    def test_rflag_chanavg(self):
-3486
+        '''flagdata: rflag with chan average and compare vs mstransform'''
-3487
-3488
+        # Unflag the original input data
-3489
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-3490
-3491
+        # STEP 1: Chan average with mstransform, then flagging with normal rflag
-3492
+        mstransform(vis=self.vis,outputvis='test_rflag_chanavg_step1.ms',datacolumn='data',
-3493
+                    chanaverage=True,chanbin=2)
-3494
+        flagdata(vis='test_rflag_chanavg_step1.ms',flagbackup=False, mode='rflag',extendflags=False)
-3500
+        # STEP 2: Flagging with rflag using time average, then time average with mstransform
-3501
+        flagdata(vis=self.vis, flagbackup=False, mode='rflag', datacolumn='DATA',
-3502
+                 channelavg=True, chanbin=2,extendflags=False)
-3503
+        mstransform(vis=self.vis, outputvis='test_rflag_chanavg_step2.ms',datacolumn='data',
-3504
+                    chanaverage=True,chanbin=2)
-3505
+        res2 = flagdata(vis='test_rflag_chanavg_step2.ms', mode='summary', spwchan=True)
-3506
-3507
+        # Compare results
-3508
+        self.assertEqual(res1['flagged'], res2['flagged'])
-3509
--
+    def test_rflag_chanavg_on_residual(self):
--
+        '''flagdata: rflag with chanavg on residual (corrected - model), and compare
--
+        vs mstransform + rflag without average'''
--
--
+        # One that has corrected
--
+        vis = "Four_ants_3C286.ms"
--
+        # Initial integration time is 1s
--
+        timebin = '8s'
--
--
+        # Unflag the original input data
--
+        flagdata(vis, flagbackup=False, mode='unflag')
--
--
+        # using action calculate, which is faster (reduced I/O) and enough to test thresholds
--
+        # using only these spws for speed
--
+        spws = '0,1,2'
--
--
+        # STEP 1: rflag-calculate with original MS
--
+        res1 = flagdata(vis=vis, spw=spws, action='calculate', mode='rflag',
--
+                        datacolumn='residual', extendflags=False)
--
--
+        # STEP 2: chanavg with mstransform, then rflag-calculate on residual
--
+        flagged1 = 'test_rflag_timeavg_residual_step2.ms'
--
+        mstransform(vis=vis, spw=spws, outputvis=flagged1,
--
+                    datacolumn='data', chanaverage=True, chanbin=32)
--
+        res2 = flagdata(vis=vis, spw=spws, action='calculate', mode='rflag',
--
+                        datacolumn='residual', extendflags=False)
--
--
+        # STEP 3: rflag-calculate with chanavg on residual
--
+        res3 = flagdata(vis=vis, spw=spws, action='calculate', mode='rflag',
--
+                        datacolumn='residual', channelavg=True, chanbin=32,
--
+                        extendflags=False)
--
--
+        def check_reports_chanavg(report1, report2, report3):
--
+            import numpy as np
--
+            self.assertEqual(report2['type'], 'rflag')
--
+            self.assertEqual(report3['type'], report2['type'])
--
+            freq_tol = 1e-1
--
+            self.assertTrue(np.allclose(report1['freqdev'], report2['freqdev'],
--
+                                        rtol=freq_tol))
--
+            # divide 3rd column (thresholds). Matrices have rows like: field, spw, threshold.
--
+            freq_div_tol = 2.5e-1
--
+            report2['freqdev'][:,2] = report2['freqdev'][:,2] / 2
--
+            self.assertTrue(np.allclose(report2['freqdev'], report3['freqdev'],
--
+                                        rtol=freq_div_tol))
--
+            time_tol = 1e-1
--
+            self.assertTrue(np.allclose(report1['timedev'], report2['timedev'],
--
+                                        rtol=time_tol))
--
+            time_div_tol = 6e-1
--
+            report1['timedev'][:,2] = report1['timedev'][:,2] / 4
--
+            self.assertTrue(np.allclose(report1['timedev'],
--
+                                        report3['timedev'], rtol=time_div_tol))
--
--
+        self.assertEqual(res1['type'], 'list')
--
+        self.assertEqual(res1['type'], res2['type'])
--
+        self.assertEqual(res1['type'], res3['type'])
--
+        check_reports_chanavg(res1['report0'], res2['report0'], res3['report0'])
--
-3510
+    def test_rflag_time_chanavg(self):
-3511
+        '''flagdata: rflag with time/chan average and compare vs mstransform'''
-3512
-3513
+        # Unflag the original input data
-3514
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-3515
-3516
+        # STEP 1: Chan average with mstransform, then flagging with normal rflag
-3517
+        mstransform(vis=self.vis,outputvis='test_rflag_time_chanavg_step1.ms',datacolumn='data',
-3518
+                    timeaverage=True,timebin='2s',chanaverage=True,chanbin=2)
-3519
+        flagdata(vis='test_rflag_time_chanavg_step1.ms',flagbackup=False, mode='rflag',
-3605
+        # STEP 2: Flagging with tfcrop using time average, then time average with mstransform
-3606
+        flagdata(vis=self.vis, flagbackup=False, mode='tfcrop', datacolumn='DATA',
-3607
+                 timeavg=True, timebin='2s', channelavg=True, chanbin=2, extendflags=False)
-3608
+        mstransform(vis=self.vis, outputvis='test_tfcrop_time_chanavg_step2.ms',datacolumn='data',
-3609
+                    timeaverage=True,timebin='2s',chanaverage=True,chanbin=2)
-3610
+        res2 = flagdata(vis='test_tfcrop_time_chanavg_step2.ms', mode='summary', spwchan=True)
-3611
-3612
+        # Compare results
-3613
+        self.assertEqual(res2['flagged'], res2['flagged'])
-3614
-+
-+
+# Motivated by CAS-11397. test_preaveraging is about datacolumn='data', and checks what
-+
+# flags are written to the output
-+
+# test_preaveraging_rflag_residual is about datacolumn='residual' and doesn't write flags. It
-+
+# checks the threshold calculations from RFlag
-+
+class test_preaveraging_rflag_residual(test_base):
-+
+    """Test pre-averaging (channel / time) with RFlag and datacolumn='residual'"""
-+
-+
+    def setUp(self):
-+
+        self.setUp_data4tfcrop()
-+
-+
+    def tearDown(self):
-+
+        os.system('rm -rf test_rflag_timeavg_residual*step2*ms')
-+
+        os.system('rm -rf test_rflag_channelavg_residual*step2*ms')
-+
-+
+    def test_rflag_timeavg_on_residual(self):
-+
+        '''flagdata: rflag with timeavg on residual (corrected - model), and compare
-+
+        vs mstransform + rflag without timeavg'''
-+
-+
+        # Initial integration time of 'Four_ants_3C286.ms' is 1s
-+
+        timebin = '8s'
-+
-+
+        # using action calculate, which is faster (reduced I/O) and enough to test thresholds
-+
+        # using only these spws for speed
-+
+        spws = '0,1,2'
-+
-+
+        # STEP 1: rflag-calculate with original time
-+
+        res1 = flagdata(vis=self.vis, spw=spws, action='calculate', mode='rflag',
-+
+                        datacolumn='residual', extendflags=False)
-+
-+
+        # STEP 2: timeavg with mstransform, then rflag-calculate on residual
-+
+        flagged2 = 'test_rflag_timeavg_residual_step2.ms'
-+
+        mstransform(vis=self.vis, spw=spws, outputvis=flagged2,
-+
+                    datacolumn='data,model,corrected', timeaverage=True, timebin=timebin)
-+
+        res2 = flagdata(vis=flagged2, spw=spws, action='calculate', mode='rflag',
-+
+                        datacolumn='residual', extendflags=False)
-+
-+
+        # STEP 3: rflag-calculate with timeavg on residual
-+
+        res3 = flagdata(vis=self.vis, spw=spws, action='calculate', mode='rflag',
-+
+                        datacolumn='residual', timeavg=True, timebin=timebin,
-+
+                        extendflags=False)
-+
-+
+        def check_reports_timeavg(report1, report2, report3):
-+
+            import numpy as np
-+
+            print "report 1", report1
-+
+            print "report 2", report2
-+
+            print "report 3", report3
-+
+            self.assertEqual(report2['type'], 'rflag')
-+
+            self.assertEqual(report3['type'], report2['type'])
-+
+            freq_tol = 1e-1
-+
+            self.assertTrue(np.allclose(report1['freqdev'], report3['freqdev'],
-+
+                                        rtol=freq_tol))
-+
+            self.assertTrue(np.allclose(report2['freqdev'], report3['freqdev'],
-+
+                                        rtol=freq_tol))
-+
+            # divide 3rd column (thresholds). Matrices have rows like: field, spw, threshold.
-+
+            report1['timedev'][:,2] = report1['timedev'][:,2] / np.sqrt(8)
-+
+            time_div_tol = 3.3e-1
-+
+            self.assertTrue(np.allclose(report1['timedev'], report3['timedev'],
-+
+                                        rtol=time_div_tol))
-+
+            time_tol = 1.5e-1
-+
+            self.assertTrue(np.allclose(report2['timedev'],
-+
+                                        report3['timedev'], rtol=time_tol))
-+
-+
+        self.assertEqual(res1['type'], 'list')
-+
+        self.assertEqual(res1['type'], res2['type'])
-+
+        self.assertEqual(res1['type'], res3['type'])
-+
+        check_reports_timeavg(res1['report0'], res2['report0'], res3['report0'])
-+
-+
+    def test_rflag_channelavg_on_residual(self):
-+
+        '''flagdata: rflag with channelavg on residual (corrected - model), and compare
-+
+        vs mstransform + rflag without average'''
-+
-+
+        # Initial integration time of 'Four_ants_3C286.ms' is 1s
-+
+        timebin = '8s'
-+
-+
+        # using action calculate, which is faster (reduced I/O) and enough to test thresholds
-+
+        # using only these spws for speed
-+
+        spws = '0,1,2'
-+
-+
+        # STEP 1: rflag-calculate with original MS
-+
+        res1 = flagdata(vis=self.vis, spw=spws, action='calculate', mode='rflag',
-+
+                        datacolumn='residual', extendflags=False)
-+
-+
+        # STEP 2: chanavg with mstransform, then rflag-calculate on residual
-+
+        flagged2 = 'test_rflag_channelavg_residual_step2.ms'
-+
+        mstransform(vis=self.vis, spw=spws, outputvis=flagged2,
-+
+                    datacolumn='data,model,corrected', chanaverage=True, chanbin=32)
-+
+        res2 = flagdata(vis=flagged2, spw=spws, action='calculate', mode='rflag',
-+
+                        datacolumn='residual', extendflags=False)
-+
-+
+        # STEP 3: rflag-calculate with channelavg on residual
-+
+        res3 = flagdata(vis=self.vis, spw=spws, action='calculate', mode='rflag',
-+
+                        datacolumn='residual', channelavg=True, chanbin=32,
-+
+                        extendflags=False)
-+
-+
+        def check_reports_channelavg(report1, report2, report3):
-+
+            import numpy as np
-+
+            self.assertEqual(report2['type'], 'rflag')
-+
+            self.assertEqual(report3['type'], report2['type'])
-+
+            # divide 3rd column (thresholds). Matrices have rows like: field, spw, threshold.
-+
+            report1['freqdev'][:,2] = report1['freqdev'][:,2] / 2
-+
+            freq_div_tol = 1e-1
-+
+            self.assertTrue(np.allclose(report1['freqdev'], report3['freqdev'],
-+
+                                        rtol=freq_div_tol))
-+
+            freq_tol = 5e-2
-+
+            self.assertTrue(np.allclose(report2['freqdev'], report3['freqdev'],
-+
+                                        rtol=freq_tol))
-+
-+
+            report1['timedev'][:,2] = report1['timedev'][:,2] / 4
-+
+            time_div_tol = 6.6e-1
-+
+            self.assertTrue(np.allclose(report1['timedev'], report3['timedev'],
-+
+                                        rtol=time_div_tol))
-+
+            time_tol = 5e-2
-+
+            self.assertTrue(np.allclose(report2['timedev'],
-+
+                                        report3['timedev'], rtol=time_tol))
-+
-+
+        self.assertEqual(res1['type'], 'list')
-+
+        self.assertEqual(res1['type'], res2['type'])
-+
+        self.assertEqual(res1['type'], res3['type'])
-+
+        check_reports_channelavg(res1['report0'], res2['report0'], res3['report0'])
-+
-+
-3737
+class test_virtual_col(test_base):
-3738
+    def setUp(self):
-3739
+        self.setUp_ngc5921(force=True)
-3740
-3741
+    def tearDown(self):
-3742
+        os.system('rm -rf ngc5921*')
-3743
-3744
+    @unittest.skip('Skip until CAS-10383 is fixed')
-3745
+    def test_no_model_col(self):
-3746
+        '''flagdata: catch failure when MODEL or virtual MODEL do not exist'''
-3809
+            test_CASA_4_0_bug_fix,
-3810
+            test_correlations,
-3811
+            test_tsys,
-3812
+            test_bandpass,
-3813
+            test_newcal,
-3814
+            test_weight_spectrum,
-3815
+            test_float_column,
-3816
+            test_tbuff,
-3817
+            TestMergeManualTimerange,
-3818
+            test_preaveraging,
-+
+            test_preaveraging_rflag_residual,
-3820
+            test_virtual_col,
-3821
+            cleanup]

casa5

Commits

Jira Issues

Add shortcut