casa / casa6 / ad5e5c3cdac

Commits

Federico Montesino Pouzols authored ad5e5c3cdac Merge 07 Jan 2022

Merge remote-tracking branch 'origin/master' into CAS-13576

casa5/gcwrap/python/scripts/tests/test_flagdata.py

Modified

-7
+import pprint
-8
+import numpy as np
-9
+from numpy import array
-10
+import ast
-11
-12
+from casatasks.private.casa_transition import is_CASA6
-13
+if is_CASA6:
-14
+    import sys
-15
-16
+    from casatasks import flagcmd, flagdata, mstransform, setjy, delmod, split
--
+    from casatools import ctsys, agentflagger, table
-+
+    from casatools import ctsys, agentflagger, table, measures, quanta
-18
+    from casatasks.private.parallel.parallel_task_helper import ParallelTaskHelper
-19
+    from casatasks.private import flaghelper as fh
-20
-21
+    ### for testhelper import
-22
+    #sys.path.append(os.path.abspath(os.path.dirname(__file__)))
-23
+    #import testhelper as th
-24
-25
+    # CASA6 doesn't need defaul
-26
+    def default(atask):
-27
+        pass
-28
-29
+    ctsys_resolve = ctsys.resolve
-30
+else:
-31
+    from tasks import flagcmd, flagdata, mstransform, setjy, delmod, split
-32
+    from taskinit import aftool as agentflagger
-33
+    from taskinit import tbtool as table
-+
+    from taskinit import metool as measures
-+
+    from taskinit import qatool as quanta
-36
+    from __main__ import default
-37
+    from parallel.parallel_task_helper import ParallelTaskHelper
-38
+    import flaghelper as fh
-39
+    #import testhelper as th
-40
-41
+    def ctsys_resolve(apath):
-42
+        dataPath = os.path.join(os.environ['CASAPATH'].split()[0],'casatestdata/')
-43
+        return os.path.join(dataPath,apath)
-44
-45
+from casatestutils import testhelper as th
-290
-291
+    def setUp_data4preaveraging(self):
-292
+        # Four_ants_3C286_spw9_small_for_preaveraging.ms was generated with a command like:
-293
+        # mstransform(vis='Four_ants_3C286.ms',
-294
+        #             outputvis='Four_ants_3C286_spw9_small_for_preaveraging.ms',
-295
+        #             datacolumn='data',spw='9', antenna='1&2',
-296
+        #             timerange='2010/10/16/14:45:08.50~2010/10/16/14:45:11.50')
-297
+        self.vis = 'Four_ants_3C286_spw9_small_for_preaveraging.ms'
-298
+        self._check_path_move_remove_versions_unflag_etc()
-299
-+
+    def setUp_data4timeavg(self):
-+
+        # Four_ants_3C286_spw9_small_for_preaveraging.ms was generated with a command like:
-+
+        # The different wrt data4preaveraging is that we include more rows / integrations so
-+
+        # that it is possible to run timeavg with a bigger timebin (up to 100s).
-+
+        # mstransform(vis='Four_ants_3C286.ms',
-+
+        #             outputvis='Four_ants_3C286_spw9_small_for_timeavg.ms',
-+
+        #             datacolumn='data',spw='9', antenna='1&2',
-+
+        #             timerange='2010/10/16/14:45:08.50~2010/10/16/14:46:48.50')
-+
+        self.vis = 'Four_ants_3C286_spw9_small_for_timeavg.ms'
-+
+        self._check_path_move_remove_versions_unflag_etc()
-+
-311
+    def setUp_shadowdata1(self):
-312
+        '''ALMA ACA observation with one field in APP ref frame'''
-313
+        self.vis = "shadowAPP.ms"
-314
-315
+        if os.path.exists(self.vis):
-316
+            print("The MS is already around, just unflag")
-317
+        else:
-318
+            print("Moving data...")
-319
+            os.system('cp -RH '+os.path.join(datapath,self.vis)+' '+ self.vis)
-320
-2379
+        # should not flag anything
-2380
+        res = flagdata(vis=self.vis, mode='summary', spw='5,9,10,11')
-2381
+        self.assertEqual(res['flagged'], 0)
-2382
-2383
+    def test_residual_col(self):
-2384
+        '''flagdata: clip RESIDUAL column'''
-2385
+        flagdata(vis=self.vis, flagbackup=False, mode='clip', datacolumn='RESIDUAL', clipzeros=True)
-2386
+        res = flagdata(vis=self.vis, mode='summary')
-2387
+        self.assertEqual(res['flagged'], 137472)
-2388
--
+    def test_timeavg1(self):
-+
+    def test_clip_timeavg_cmp_mstransform(self):
-2390
+        '''flagdata: clip with time average and compare with mstransform'''
-+
-+
+        def check_expected_flag_positions(self, msname):
-+
+            """
-+
+            Implements a very hand-made check on the exact positions of flags for the test
-+
+            dataset used in these tests.
-+
-+
+            This has been crossed-checked manually, looking at the visibility values and the
-+
+            thresholds set in clip (clipminmax). The pattern used here must be produced by
-+
+            the (back)propagation of flags with time average if it is working correctly.
-+
+            """
-+
+            try:
-+
+                tbt = table()
-+
+                tbt.open(msname)
-+
+                flags = tbt.getcol('FLAG')
-+
-+
+                # Indices are: correlation, channel, time:
-+
+                exp_flags = [[ 0, 51, 0], [ 0, 51, 1], [ 0, 60, 0], [ 0, 60, 1],
-+
+                             [ 1, 51, 0], [ 1, 51, 1], [ 1, 60, 1], [ 2, 51, 0],
-+
+                             [ 2, 51, 1], [ 2, 60, 0], [ 2, 60, 1], [ 3 ,21, 1],
-+
+                             [ 3, 51, 0], [ 3, 51, 1], [ 3, 60, 0], [ 3, 60, 1]]
-+
+                expected = np.full((4, 64, 2), False)
-+
+                for pos in exp_flags:
-+
+                    expected[pos[0], pos[1], pos[2]] = True
-+
-+
+                np.testing.assert_equal(flags, expected,
-+
+                                        "Flags do not match the expected, manually verified "
-+
+                                        "pattern.")
-+
+            finally:
-+
+                tbt.close()
-+
-2421
+        # Create an output with 4 rows
-2422
+        split(vis=self.vis,outputvis='timeavg.ms',datacolumn='data',spw='9',scan='30',antenna='1&2',
-2423
+               timerange='2010/10/16/14:45:08.50~2010/10/16/14:45:11.50')
-+
+        flagdata('timeavg.ms', flagbackup=False, mode='unflag')
-2425
-2426
+        # STEP 1
-2427
+        # Create time averaged output in mstransform
--
+        mstransform('timeavg.ms',outputvis='test_residual_step1_timeavg.ms',
-+
+        ms_step1 = 'test_residual_step1_timeavg.ms'
-+
+        mstransform('timeavg.ms',outputvis=ms_step1,
-2430
+                    datacolumn='data',timeaverage=True,timebin='2s')
-2431
-2432
+        # clip it
--
+        flagdata('test_residual_step1_timeavg.ms',flagbackup=False, mode='clip',
-+
+        flagdata(ms_step1, flagbackup=False, mode='clip',
-2434
+                 clipminmax=[0.0,0.08])
--
+        res1 = flagdata(vis='test_residual_step1_timeavg.ms', mode='summary', spwchan=True)
-+
+        res1 = flagdata(vis=ms_step1, mode='summary', spwchan=True)
-2436
-2437
+        # STEP 2
-2438
+        # Clip with time averaging.
--
+        flagdata('timeavg.ms', flagbackup=False, mode='unflag')
-+
+        ms_step2 = 'test_residual_step2_timeavg.ms'
-2440
+        flagdata(vis='timeavg.ms', flagbackup=False, mode='clip', datacolumn='DATA',
-2441
+                 timeavg=True, timebin='2s', clipminmax=[0.0,0.08])
-2442
-2443
+        # Do another time average in mstransform to have the corrected averaged visibilities
--
+        mstransform('timeavg.ms',outputvis='test_residual_step2_timeavg.ms',
-+
+        mstransform('timeavg.ms', outputvis=ms_step2,
-2445
+                    datacolumn='data',timeaverage=True,timebin='2s')
-2446
--
+        res2 = flagdata(vis='test_residual_step2_timeavg.ms', mode='summary', spwchan=True)
-+
+        res2 = flagdata(vis=ms_step2, mode='summary', spwchan=True)
-2448
-+
+        # Compare step1 vs step2
-2450
+        self.assertEqual(res1['flagged'], res2['flagged'])
-+
+        # Check specific channels
-2452
+        self.assertEqual(res2['spw:channel']['0:21']['flagged'], 1)
-2453
+        self.assertEqual(res2['spw:channel']['0:51']['flagged'], 8)
-2454
+        self.assertEqual(res2['spw:channel']['0:60']['flagged'], 7)
-2455
--
+    def test_timeavg2(self):
-+
+        # Additional checks on the exact positions of flags, to better cover issues found
-+
+        # in CAS-12737, CAS-12910.
-+
+        check_expected_flag_positions(self, ms_step1)
-+
+        check_expected_flag_positions(self, ms_step2)
-+
-+
+    def test_timeavg_spw9_2scans(self):
-2462
+        '''flagdata: clip with time averaging in spw 9'''
-2463
-2464
+        flagdata(vis=self.vis, flagbackup=False, mode='clip', datacolumn='DATA', spw='9',
-2465
+                 timeavg=True, timebin='2s', clipminmax=[0.0,0.08])
-2466
-2467
+        res = flagdata(vis=self.vis, mode='summary', spw='9')
--
+        self.assertEqual(res['spw']['9']['flagged'], 42370)
--
+        self.assertEqual(res['flagged'], 42370)
-+
+        self.assertEqual(res['spw']['9']['flagged'], 42106)
-+
+        self.assertEqual(res['flagged'], 42106)
-2470
-2471
-2472
+    def test_clip_no_model_col(self):
-2473
+        "flagdata: Should fail when MODEL or virtual MODEL columns do not exist"
-2474
+        # Use an MS without MODEL_DATA column
-2475
+        self.setUp_ngc5921(True)
-2476
-2477
+        # RESIDUAL = CORRECTED - MODEL.
-2478
+        # It should fail and not flag anything
-2479
+        datacols = ["RESIDUAL","RESIDUAL_DATA"]
-3793
+        self.assertEqual(res[6]['mode'], 'summary3')
-3794
-3795
-3796
+class test_preaveraging(test_base):
-3797
+    """Test channel/time pre-averaging for visibility-based flagging"""
-3798
-3799
+    def setUp(self):
-3800
+        self.setUp_data4preaveraging()
-3801
+        self.corrs = ['RL', 'LL', 'LR', 'RR']
-3802
--
+    def tearDown(self):
-+
+    def tearDown(self):
-3804
+        os.system('rm -rf test_preaveraging.ms')
-3805
+        os.system('rm -rf test_clip_timeavg*')
-3806
+        os.system('rm -rf test_clip_chanavg*')
-3807
+        os.system('rm -rf test_clip_time_chanavg*')
-3808
+        os.system('rm -rf test_rflag_timeavg*')
-3809
+        os.system('rm -rf test_rflag_chanavg*')
-3810
+        os.system('rm -rf test_rflag_time_chanavg*')
-3811
+        os.system('rm -rf test_tfcrop_timeavg*')
-3812
+        os.system('rm -rf test_tfcrop_chanavg*')
-3813
+        os.system('rm -rf test_tfcrop_time_chanavg*')
-3814
-3815
+    def test_clip_timeavg(self):
-3816
+        '''flagdata: clip with time average and compare vs mstransform'''
-3817
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
+        # Unflag the original input data - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-3820
-3821
+        # STEP 1: Time average with mstransform, then flagging with normal clip
-3822
+        mstransform(vis=self.vis,outputvis='test_clip_timeavg_step1.ms',datacolumn='data',
-3823
+                    timeaverage=True,timebin='2s')
-3824
+        flagdata(vis='test_clip_timeavg_step1.ms',flagbackup=False, mode='clip',clipminmax=[0.0,0.08])
-3825
+        res1 = flagdata(vis='test_clip_timeavg_step1.ms', mode='summary', spwchan=True)
-3826
-3827
+        # Unflag the original input data
-3828
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-3829
-3833
+        mstransform(vis=self.vis,outputvis='test_clip_timeavg_step2.ms',datacolumn='data',
-3834
+                    timeaverage=True,timebin='2s')
-3835
+        res2 = flagdata(vis='test_clip_timeavg_step2.ms', mode='summary', spwchan=True)
-3836
-3837
+        # Compare results
-3838
+        self.assertEqual(res1['flagged'], res2['flagged'])
-3839
-3840
+    def test_clip_chanavg(self):
-3841
+        '''flagdata: clip with chan average and compare vs mstransform'''
-3842
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
+        # Unflag the original input data - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-3845
-3846
+        # STEP 1: Chan average with mstransform, then flagging with normal clip
-3847
+        mstransform(vis=self.vis,outputvis='test_clip_chanavg_step1.ms',datacolumn='data',
-3848
+                    chanaverage=True,chanbin=2)
-3849
+        flagdata(vis='test_clip_chanavg_step1.ms',flagbackup=False, mode='clip',clipminmax=[0.0,0.08])
-3850
+        res1 = flagdata(vis='test_clip_chanavg_step1.ms', mode='summary', spwchan=True)
-3851
-3852
+        # Unflag the original input data
-3853
+        flagdata(vis=self.vis, flagbackup=False, mode='unflag')
-3854
-3858
+        mstransform(vis=self.vis, outputvis='test_clip_chanavg_step2.ms',datacolumn='data',
-3859
+                    chanaverage=True,chanbin=2)
-3860
+        res2 = flagdata(vis='test_clip_chanavg_step2.ms', mode='summary', spwchan=True)
-3861
-3862
+        # Compare results
-3863
+        self.assertEqual(res1['flagged'], res2['flagged'])
-3864
-3865
+    def test_clip_time_chanavg(self):
-3866
+        '''flagdata: clip with time/chan average and compare vs mstransform'''
-3867
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
+        # Unflag the original input data  - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-3870
-3871
+        # STEP 1: Chan average with mstransform, then flagging with normal clip
-3872
+        mstransform(vis=self.vis,outputvis='test_clip_time_chanavg_step1.ms',datacolumn='data',
-3873
+                    timeaverage=True,timebin='2s',chanaverage=True,chanbin=2)
-3874
+        flagdata(vis='test_clip_time_chanavg_step1.ms',flagbackup=False, mode='clip',clipminmax=[0.0,0.08])
-3875
+        res1 = flagdata(vis='test_clip_time_chanavg_step1.ms', mode='summary', spwchan=True)
-3876
-3877
+        # Unflag the original input data
-3878
+        flagdata(vis=self.vis, flagbackup=False, mode='unflag')
-3879
-3880
+        # STEP 2: Flagging with clip using time average, then time average with mstransform
-3881
+        flagdata(vis=self.vis, flagbackup=False, mode='clip', datacolumn='DATA',
-3882
+                 timeavg=True, timebin='2s', channelavg=True, chanbin=2, clipminmax=[0.0,0.08])
-3883
+        mstransform(vis=self.vis, outputvis='test_clip_time_chanavg_step2.ms',datacolumn='data',
-3884
+                    timeaverage=True,timebin='2s',chanaverage=True,chanbin=2)
-3885
+        res2 = flagdata(vis='test_clip_time_chanavg_step2.ms', mode='summary', spwchan=True)
-3886
-3887
+        # Compare results
--
+        self.assertEqual(res1['flagged'], res2['flagged'])
-+
+        self.assertEqual(res1['flagged'], res2['flagged'])
-3889
-3890
+    def test_rflag_timeavg(self):
-3891
+        '''flagdata: rflag with time average and compare vs mstransform'''
-3892
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
+        # # Unflag the original input data - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-3895
-3896
+        # STEP 1: Time average with mstransform, then flagging with normal rflag
-3897
+        mstransform(vis=self.vis,outputvis='test_rflag_timeavg_step1.ms',datacolumn='data',
-3898
+                    timeaverage=True,timebin='2s')
-3899
+        flagdata(vis='test_rflag_timeavg_step1.ms',flagbackup=False, mode='rflag',extendflags=False)
-3900
+        res1 = flagdata(vis='test_rflag_timeavg_step1.ms', mode='summary', spwchan=True)
--
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
-+
+        # # Unflag the original input data - not needed
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-3904
-3905
+        # STEP 2: Flagging with rflag using time average, then time average with mstransform
-3906
+        flagdata(vis=self.vis, flagbackup=False, mode='rflag', datacolumn='DATA',
-3907
+                 timeavg=True, timebin='2s', extendflags=False)
-3908
+        mstransform(vis=self.vis,outputvis='test_rflag_timeavg_step2.ms',datacolumn='data',
-3909
+                    timeaverage=True,timebin='2s')
-3910
+        res2 = flagdata(vis='test_rflag_timeavg_step2.ms', mode='summary', spwchan=True)
-3911
-3912
+        # Compare results
--
+        self.assertEqual(res1['total'], res2['total'])
-+
+        self.assertEqual(res1['flagged'], res2['flagged'])
-3914
+        self.assertEqual(res1['flagged'], 27)
--
+        self.assertEqual(res2['flagged'], 20)
-+
+        for cor in self.corrs:
-+
+            self.assertEqual(res1['correlation'][cor]['flagged'],
-+
+                             res2['correlation'][cor]['flagged'])
-3918
-3919
+    def test_rflag_timeavg_extendflags(self):
-3920
+        '''flagdata: rflag with time average + extendflags, and compare vs mstransform'''
-3921
+        # Unflag the original input data
-3922
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-3923
-3924
+        timebin = '2s'
-3925
-3926
+        # STEP 1: Time average with mstransform, then flagging with normal rflag+extendflags
-3927
+        mstransform(vis=self.vis, outputvis='test_rflag_timeavg_extendflags_step1.ms',
-3935
-3936
+        # STEP 2: Flagging with rflag using time average, then time average with mstransform
-3937
+        flagdata(vis=self.vis, flagbackup=False, mode='rflag', datacolumn='DATA',
-3938
+                 timeavg=True, timebin='2s', extendflags=True)
-3939
+        mstransform(vis=self.vis, outputvis='test_rflag_timeavg_extendflags_step2.ms',
-3940
+                    datacolumn='data', timeaverage=True, timebin=timebin)
-3941
+        res2 = flagdata(vis='test_rflag_timeavg_extendflags_step2.ms', mode='summary')
-3942
-3943
+        # Compare results
-3944
+        self.assertEqual(res1['total'], res2['total'])
-+
+        self.assertEqual(res1['flagged'], res2['flagged'])
-3946
+        self.assertEqual(res1['flagged'], 40)
--
+        self.assertEqual(res2['flagged'], 24)
-3947
+        for cor in self.corrs:
-3948
+            self.assertEqual(res1['correlation'][cor]['flagged'],10)
--
+            self.assertEqual(res2['correlation'][cor]['flagged'],6)
-+
+            self.assertEqual(res1['correlation'][cor]['flagged'],
-+
+                             res2['correlation'][cor]['flagged'])
-3951
-3952
+    def test_rflag_chanavg(self):
-3953
+        '''flagdata: rflag with chan average and compare vs mstransform'''
-3954
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
+        # Unflag the original input data  - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-3957
-3958
+        # STEP 1: Chan average with mstransform, then flagging with normal rflag
-3959
+        mstransform(vis=self.vis,outputvis='test_rflag_chanavg_step1.ms',datacolumn='data',
-3960
+                    chanaverage=True,chanbin=2)
-3961
+        flagdata(vis='test_rflag_chanavg_step1.ms',flagbackup=False, mode='rflag',extendflags=False)
-3962
+        res1 = flagdata(vis='test_rflag_chanavg_step1.ms', mode='summary', spwchan=True)
-3963
-3964
+        # Unflag the original input data
-3965
+        flagdata(vis=self.vis, flagbackup=False, mode='unflag')
-3966
-3972
+        res2 = flagdata(vis='test_rflag_chanavg_step2.ms', mode='summary', spwchan=True)
-3973
-3974
+        # Compare results
-3975
+        self.assertEqual(res1['flagged'], res2['flagged'])
-3976
+        for cor in self.corrs:
-3977
+            self.assertEqual(res2['correlation'][cor]['flagged'],
-3978
+                             res1['correlation'][cor]['flagged'])
-3979
-3980
+    def test_rflag_chanavg_extendflags(self):
-3981
+        '''flagdata: rflag with chan average + extendflags, and compare vs mstransform'''
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
-+
+        # Unflag the original input data  - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-3985
-3986
+        chanbin = 8
-3987
-3988
+        # STEP 1: Chan average with mstransform, then flagging with normal rflag
-3989
+        mstransform(vis=self.vis, outputvis='test_rflag_chanavg_extendflags_step1.ms',
-3990
+                    datacolumn='data', chanaverage=True, chanbin=chanbin)
-3991
+        flagdata(vis='test_rflag_chanavg_extendflags_step1.ms', flagbackup=False,
-3992
+                 mode='rflag', extendflags=True)
-3993
+        res1 = flagdata(vis='test_rflag_chanavg_extendflags_step1.ms', mode='summary')
-3994
-4007
+        self.assertEqual(res1['flagged'], res2['flagged'])
-4008
+        self.assertEqual(res1['flagged'], 20)
-4009
+        for cor in self.corrs:
-4010
+            self.assertEqual(res1['correlation'][cor]['flagged'], 5)
-4011
+            self.assertEqual(res2['correlation'][cor]['flagged'],
-4012
+                             res1['correlation'][cor]['flagged'])
-4013
-4014
+    def test_rflag_time_chanavg(self):
-4015
+        '''flagdata: rflag with time/chan average and compare vs mstransform'''
-4016
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
+        # Unflag the original input data - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-4019
-4020
+        # STEP 1: chan+time average with mstransform, then flagging with normal rflag
-4021
+        mstransform(vis=self.vis,outputvis='test_rflag_time_chanavg_step1.ms',
-4022
+                    datacolumn='data', timeaverage=True, timebin='2s',
-4023
+                    chanaverage=True, chanbin=2)
-4024
+        res1 = flagdata(vis='test_rflag_time_chanavg_step1.ms', action='calculate',
-4025
+                        mode='rflag', extendflags=False)
-4026
-4027
+        # STEP 2: rflag using chan+time average, then mstransform using chan+time avg
-4028
+        flagdata(vis=self.vis, flagbackup=False, mode='rflag', datacolumn='DATA',
-4043
+        tolerances = [1.1, 7.5e-1]
-4044
+        for threshold_type, tol in zip(['freqdev', 'timedev'], tolerances):
-4045
+            self.assertTrue(np.less_equal(res2['report0'][threshold_type],
-4046
+                                          res1['report0'][threshold_type]).all())
-4047
+            self.assertTrue(np.allclose(res1['report0'][threshold_type],
-4048
+                                        res2['report0'][threshold_type], rtol=tol))
-4049
-4050
+    def test_tfcrop_timeavg(self):
-4051
+        '''flagdata: tfcrop with time average and compare vs mstransform'''
-4052
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
+        # Unflag the original input data - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-4055
-4056
+        # STEP 1: Time average with mstransform, then flagging with normal tfcrop
-4057
+        mstransform(vis=self.vis,outputvis='test_tfcrop_timeavg_step1.ms',datacolumn='data',
-4058
+                    timeaverage=True,timebin='2s')
-4059
+        flagdata(vis='test_tfcrop_timeavg_step1.ms',flagbackup=False, mode='tfcrop',
-4060
+                 extendflags=False)
-4061
+        res1 = flagdata(vis='test_tfcrop_timeavg_step1.ms', mode='summary', spwchan=True)
-4062
-4063
+        # Unflag the original input data
-4064
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-4065
-4066
+        # STEP 2: Flagging with tfcrop using time average, then time average with mstransform
-4067
+        flagdata(vis=self.vis, flagbackup=False, mode='tfcrop', datacolumn='DATA',
-4068
+                 timeavg=True, timebin='2s', extendflags=False)
-4069
+        mstransform(vis=self.vis,outputvis='test_tfcrop_timeavg_step2.ms',datacolumn='data',
-4070
+                    timeaverage=True,timebin='2s')
-4071
+        res2 = flagdata(vis='test_tfcrop_timeavg_step2.ms', mode='summary', spwchan=True)
-4072
-4073
+        # Check results
--
+        self.assertEqual(res1['flagged'], 36)
--
+        self.assertEqual(res2['flagged'], 60)
-+
+        self.assertEqual(res2['flagged'], res2['flagged'])
-+
+        for cor in self.corrs:
-+
+            self.assertEqual(res2['correlation'][cor]['flagged'],
-+
+                             res1['correlation'][cor]['flagged'])
-4078
-4079
+    def test_tfcrop_timeavg_extendflags(self):
-4080
+        '''flagdata: tfcrop with time average + extendflags, and compare vs mstransform'''
-4081
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
+        # Unflag the original input data  - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-4084
-4085
+        timebin = '2s'
-4086
-4087
+        # STEP 1: Time average with mstransform, then flagging with normal tfcrop
-4088
+        mstransform(vis=self.vis, outputvis='test_tfcrop_timeavg_extendflags_step1.ms',
-4089
+                    datacolumn='data', timeaverage=True, timebin=timebin)
-4090
+        flagdata(vis='test_tfcrop_timeavg_extendflags_step1.ms', flagbackup=False,
-4091
+                 mode='tfcrop', extendflags=True)
-4092
+        res1 = flagdata(vis='test_tfcrop_timeavg_extendflags_step1.ms', mode='summary')
-4093
-4096
-4097
+        # STEP 2: Flagging with tfcrop using time average, then time average with mstransform
-4098
+        flagdata(vis=self.vis, flagbackup=False, mode='tfcrop', datacolumn='DATA',
-4099
+                 timeavg=True, timebin=timebin, extendflags=True)
-4100
+        mstransform(vis=self.vis,outputvis='test_tfcrop_timeavg_extendflags_step2.ms',
-4101
+                    datacolumn='data', timeaverage=True, timebin='2s')
-4102
+        res2 = flagdata(vis='test_tfcrop_timeavg_extendflags_step2.ms', mode='summary')
-4103
-4104
+        # Check results
-4105
+        self.assertEqual(res1['total'], res2['total'])
-+
+        self.assertEqual(res1['flagged'], res2['flagged'])
-4107
+        self.assertEqual(res1['flagged'], 96)
--
+        self.assertEqual(res2['flagged'], 128)
-4108
+        for cor in self.corrs:
-4109
+            self.assertEqual(res1['correlation'][cor]['flagged'], 24)
--
+            self.assertEqual(res2['correlation'][cor]['flagged'], 32)
-+
+            self.assertEqual(res1['correlation'][cor]['flagged'],
-+
+                             res2['correlation'][cor]['flagged'])
-4112
-4113
+    def test_tfcrop_chanavg(self):
-4114
+        '''flagdata: tfcrop with chan average and compare vs mstransform'''
-4115
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
+        # Unflag the original input data - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-4118
-4119
+        chanbin = 2
-4120
-4121
+        # STEP 1: Chan average with mstransform, then flagging with normal tfcrop
-4122
+        mstransform(vis=self.vis,outputvis='test_tfcrop_chanavg_step1.ms',datacolumn='data',
-4123
+                    chanaverage=True,chanbin=2)
-4124
+        flagdata(vis='test_tfcrop_chanavg_step1.ms',flagbackup=False, mode='tfcrop',
-4125
+                 extendflags=False)
-4126
+        res1 = flagdata(vis='test_tfcrop_chanavg_step1.ms', mode='summary', spwchan=True)
-4127
-4137
-4138
+        # Compare results
-4139
+        self.assertEqual(res1['flagged'], res2['flagged'])
-4140
+        for cor in self.corrs:
-4141
+            self.assertEqual(res2['correlation'][cor]['flagged'],
-4142
+                             res1['correlation'][cor]['flagged'])
-4143
-4144
+    def test_tfcrop_chanavg_extendflags(self):
-4145
+        '''flagdata: tfcrop with chan average + extendflags, and compare vs mstransform'''
-4146
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
-+
+        # Unflag the original input data - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-4149
-4150
+        chanbin = 4
-4151
-4152
+        # STEP 1: Chan average with mstransform, then flagging with normal tfcrop
-4153
+        mstransform(vis=self.vis, outputvis='test_tfcrop_chanavg_extendflags_step1.ms',
-4154
+                    datacolumn='data', chanaverage=True, chanbin=chanbin)
-4155
+        flagdata(vis='test_tfcrop_chanavg_extendflags_step1.ms', flagbackup=False,
-4156
+                 mode='tfcrop', extendflags=True)
-4157
+        res1 = flagdata(vis='test_tfcrop_chanavg_extendflags_step1.ms', mode='summary')
-4158
-4170
+        self.assertEqual(res1['total'], res2['total'])
-4171
+        self.assertEqual(res1['flagged'], res2['flagged'])
-4172
+        for cor in self.corrs:
-4173
+            self.assertEqual(res1['correlation'][cor]['flagged'], 8)
-4174
+            self.assertEqual(res2['correlation'][cor]['flagged'],
-4175
+                             res1['correlation'][cor]['flagged'])
-4176
-4177
+    def test_tfcrop_time_chanavg(self):
-4178
+        '''flagdata: tfcrop with time/chan average and compare vs mstransform'''
-4179
--
+        # Unflag the original input data
--
+        flagdata(self.vis, flagbackup=False, mode='unflag')
--
-+
+        # Unflag the original input data - alread done by setUp
-+
+        # flagdata(self.vis, flagbackup=False, mode='unflag')
-+
-4183
+        # STEP 1: Chan average with mstransform, then flagging with normal tfcrop
--
+        mstransform(vis=self.vis,outputvis='test_tfcrop_time_chanavg_step1.ms',datacolumn='data',
--
+                    timeaverage=True,timebin='2s',chanaverage=True,chanbin=2)
--
+        flagdata(vis='test_tfcrop_time_chanavg_step1.ms',flagbackup=False, mode='tfcrop',extendflags=False)
-+
+        mstransform(vis=self.vis,outputvis='test_tfcrop_time_chanavg_step1.ms',
-+
+                    datacolumn='data', timeaverage=True, timebin='2s', chanaverage=True,
-+
+                    chanbin=2)
-+
+        flagdata(vis='test_tfcrop_time_chanavg_step1.ms',flagbackup=False, mode='tfcrop',
-+
+                 extendflags=False)
-4189
+        res1 = flagdata(vis='test_tfcrop_time_chanavg_step1.ms', mode='summary', spwchan=True)
-4190
-4191
+        # Unflag the original input data
-4192
+        flagdata(vis=self.vis, flagbackup=False, mode='unflag')
-4193
-4194
+        # STEP 2: Flagging with tfcrop using time average, then time average with mstransform
-4195
+        flagdata(vis=self.vis, flagbackup=False, mode='tfcrop', datacolumn='DATA',
-4196
+                 timeavg=True, timebin='2s', channelavg=True, chanbin=2, extendflags=False)
-4197
+        mstransform(vis=self.vis, outputvis='test_tfcrop_time_chanavg_step2.ms',datacolumn='data',
-4198
+                    timeaverage=True,timebin='2s',chanaverage=True,chanbin=2)
-4199
+        res2 = flagdata(vis='test_tfcrop_time_chanavg_step2.ms', mode='summary', spwchan=True)
-4200
-4201
+        # Compare results
-4202
+        self.assertEqual(res2['flagged'], res2['flagged'])
-+
+        for cor in self.corrs:
-+
+            self.assertEqual(res2['correlation'][cor]['flagged'],
-+
+                             res1['correlation'][cor]['flagged'])
-4206
-4207
-4208
+# Motivated by CAS-11397. test_preaveraging is about datacolumn='data', and checks what
-4209
+# flags are written to the output
-4210
+# test_preaveraging_rflag_residual is about datacolumn='residual' and doesn't write flags. It
-4211
+# checks the threshold calculations from RFlag
-4212
+class test_preaveraging_rflag_residual(test_base):
-4213
+    """Test pre-averaging (channel / time) with RFlag and datacolumn='residual'"""
-4214
-4215
+    def setUp(self):
-4372
+        # Run setjy to create a normal MODEl column (SOURCE_MODEL)
-4373
+        setjy(vis=self.vis, field='1331+305*',modimage='',standard='Perley-Taylor 99',
-4374
+                scalebychan=False, usescratch=True)
-4375
-4376
+        flagdata(vis=self.vis,mode='clip',datacolumn='RESIDUAL_DATA',clipminmax=[2.3,3.1],clipoutside=False)
-4377
+        res = flagdata(vis=self.vis, mode='summary')['flagged']
-4378
-4379
+        self.assertEqual(res_virtual, res, 'Flagging using virtual MODEL column differs from normal MODEL column')
-4380
-4381
-+
+class test_flags_propagation_base(test_base):
-+
+    """
-+
+    Common methods and infrastructure used in test_flags_propagation_channelavg and
-+
+    test_flags_propagation_timeavg.
-+
+    """
-+
-+
+    def tearDown(self):
-+
+        shutil.rmtree(self.vis)
-+
-+
+    def get_flags(self, mss):
-+
+        """
-+
+        Returns the flags column of an MS. Use only on tiny MSs as the one used in this
-+
+        test
-+
-+
+        :param mss: An MS
-+
+        :return: The FLAG column of the MS
-+
+        """
-+
+        try:
-+
+            tbt = table()
-+
+            tbt.open(mss)
-+
+            flags = tbt.getcol('FLAG')
-+
+            return flags
-+
+        finally:
-+
+            tbt.close()
-+
-+
+    def check_flags_preserved(self, flags_before, flags_after):
-+
+        """
-+
+        Check 'flags before' against 'flags after' and ensures that all the flags set
-+
+        'before' are also set 'after'.
-+
+        The flags are expected in the same format as returned by tbtool.getcol('FLAG').
-+
+        This is to ensure the desired behavior from CAS-12737 (never lose flags).
-+
-+
+        :param before_flags: flags before manipulating/flagging an MS
-+
+        :param after_flags: flags after manipulating/flagging an MS
-+
+        :return: true if all flags set in flags_before are also set in flags_after
-+
+        """
-+
+        flag_cnt_before = np.count_nonzero(flags_before)
-+
+        and_flags = np.logical_and(flags_before, flags_after)
-+
+        flag_cnt_and = np.count_nonzero(and_flags)
-+
-+
+        if flag_cnt_and != flag_cnt_before:
-+
+            print(' * Not all the flags set before ({}) are set after ({}). Flags before: '
-+
+                  '{}\n Flags after: {}'.format(flag_cnt_before, flag_cnt_and,
-+
+                                                flags_before, flags_after))
-+
+        return flag_cnt_and == flag_cnt_before
-+
-+
-+
+@unittest.skipIf(False,
-+
+                 "These tests were added in CAS-12737. Not clear what would be the right"
-+
+                 "place for them.")
-+
+class test_flags_propagation_channelavg(test_flags_propagation_base):
-+
+    """
-+
+    Tests on the number and positions of flags when using
-+
+       flagdata + channelavg + autoflag_methods AND the dataset is already flagged
-+
+    ... where channelavg is implemented via the ChannelAverageTVI
-+
+    This is to make sure that flags set before the flagdata command are preserved
-+
+    (CAS-12737). The tests check the expected number of flags from several methods (clip,
-+
+    tfcrop, rflag) and that all the data points originally flagged are still flagged after
-+
+    applying, in the exact same positions.
-+
-+
+    Uses the small VLA dataset from "data4preaveraging" which is convenient for visual
-+
+    and/or manual inspection via the browser, table tool, etc.
-+
-+
+    To illustrate the potential "loss" of flags before the fix from CAS-12737, the tests
-+
+    use a range of chanbin values (~2...5) with intentionally sparse "a priori" flags like
-+
+    X 0 X 0 X 0 X 0     (with chanbin=2 could produce a total loss of "a priori" flags)
-+
+    or
-+
+    X 0 0 X 0 0 X 0 0   (with chanbin=3 could produce a total loss of "a priori" flags)
-+
+    ...
-+
+    There are notes in the comments that give the final number of flags that would be seen
-+
+    before the fix from CAS-12737 (much lower, lower than the original "a priori" flags).
-+
+    """
-+
-+
+    def setUp(self):
-+
+        self.setUp_data4preaveraging()
-+
-+
+    def run_auto_flag_preavg_propagation(self, chanbin=2, mode='clip', ims='', **kwargs):
-+
+        """
-+
+        Enables channel average and prepares a priori flags in a sparse pattern across
-+
+        channels such that we can test (back)propagation of flags after channel-averaging.
-+
+        One channel is flagged every 'chanbin'
-+
+        With chanbin=2, 50% of channels will be a priori flagged (X 0 X 0 X 0...)
-+
+        With chanbin-3, 33% of channels will be a priori flagged, and so on (X 0 0 X 0 0...)
-+
+        This would maximize the "loss" of flags as seen in CAS-12737
-+
-+
+        :param chanbin: chanbin as used in flagdata
-+
+        :param mode: auto-flag mode
-+
+        :param kwargs: Use kwargs to pass mode specific parameters, such as clipminmax for
-+
+        clip, etc.
-+
+        :return: res+apriori_flags+final_flags. res is the flagdata summary dict from the
-+
+        MS after applying flagging with channelavg. apriori_flags is the FLAG column before
-+
+        applying channelavg+autoflag_method. final_flags is the FLAG column after applying
-+
+        channelavg+autoflag_method.
-+
+        """
-+
+        def get_nchan(ims):
-+
+            """
-+
+            This function assumes single-SPW (as is the case in this test) or all SPWs
-+
+            have the same number of channels.
-+
-+
+            :param ims: an MS name
-+
+            :return: number of channels in SPW(s)
-+
+            """
-+
+            try:
-+
+                tbt = table()
-+
+                tbt.open(os.path.join(ims, 'SPECTRAL_WINDOW'))
-+
+                chans = tbt.getcol('NUM_CHAN')
-+
+                if len(chans) < 1:
-+
+                    raise RuntimeError('Inconsistency found, NUM_CHAN: {}'.format(chans))
-+
+                if not np.all(chans[0] == chans):
-+
+                    raise RuntimeError('This supports only MSs with all SPWs with the same '
-+
+                                       'number of channels. Got NUM_CHAN: {}'.format(chans))
-+
+                nchan = chans[0]
-+
+            except RuntimeError as exc:
-+
+                raise RuntimeError('Error while trying to figure out the #channels: {}'.
-+
+                                   format(exc))
-+
+            finally:
-+
+                tbt.close()
-+
-+
+            return nchan
-+
-+
+        flagdata(vis=ims, mode='unflag')
-+
-+
+        # Pre-flag channels, for example '*:0,1,2,4,...62'
-+
+        nchan = get_nchan(ims)
-+
+        flag_chans = np.arange(0, nchan, chanbin)
-+
+        flag_spw_str = '*:{}'.format(';'.join(['{}'.format(chan) for chan in flag_chans]))
-+
+        flagdata(vis=ims, mode='manual', spw=flag_spw_str)
-+
-+
+        apriori_flags = self.get_flags(self.vis)
-+
-+
+        res_avg = flagdata(vis=ims, mode=mode, channelavg=True, chanbin=chanbin, **kwargs)
-+
-+
+        res = flagdata(vis=ims, mode='summary')
-+
-+
+        final_flags = self.get_flags(self.vis)
-+
-+
+        return res, apriori_flags, final_flags
-+
-+
+    def test_propagation_clip_chanbin_2(self):
-+
+        """ clip, chanavg, chanbin=2, propagate flags forth and back """
-+
-+
+        # Make clip flag something (if no flags are added, the flag cube is not written)
-+
+        res, apriori_flags, final_flags =\
-+
+            self.run_auto_flag_preavg_propagation(chanbin=2, ims=self.vis,
-+
+                                                  clipminmax=[0.0, 0.1])
-+
-+
+        self.assertEqual(res['total'], 1024)
-+
+        # Before CAS-12727, there is some 'loss' of flags. This would be: 44
-+
+        # Instead of >= 512 (a priori)
-+
+        self.assertEqual(res['flagged'], 534)
-+
+        self.assertTrue(self.check_flags_preserved(apriori_flags, final_flags),
-+
+                        'Not all the flags set "before" are set "after"')
-+
-+
+    def test_propagation_clip_chanbin_3(self):
-+
+        """ clip, chanavg, chanbin=3, propagate flags forth and back """
-+
-+
+        # Make clip flag something (if no flags are added, the flag cube is not written)
-+
+        res, apriori_flags, final_flags =\
-+
+            self.run_auto_flag_preavg_propagation(chanbin=3, ims=self.vis,
-+
+                                                  clipminmax=[0.001, 0.1])
-+
-+
+        self.assertEqual(res['total'], 1024)
-+
+        # Before CAS-12727, there is some 'loss' of flags. This would be: 40
-+
+        # Instead of >= 352 (a priori)
-+
+        self.assertEqual(res['flagged'], 368)
-+
+        self.assertTrue(self.check_flags_preserved(apriori_flags, final_flags),
-+
+                        'Not all the flags set "before" are set "after"')
-+
-+
+    def test_propagation_tfcrop_chanbin_4(self):
-+
+        """ tfcrop, chanavg, chanbin=4, propagate flags forth and back """
-+
-+
+        # Make tfcrop flag something (if no flags are added, the flag cube is not written)
-+
+        res, apriori_flags, final_flags =\
-+
+            self.run_auto_flag_preavg_propagation(chanbin=4, ims=self.vis,
-+
+                                                  mode='tfcrop', extendflags=False)
-+
-+
+        self.assertEqual(res['total'], 1024)
-+
+        # Before CAS-12727, there is some 'loss' of flags. This would be: 68
-+
+        # Instead of >= 256
-+
+        self.assertEqual(res['flagged'], 307)
-+
+        self.assertTrue(self.check_flags_preserved(apriori_flags, final_flags),
-+
+                        'Not all the flags set "before" are set "after"')
-+
-+
+    def test_propagation_rflag_chanbin_5(self):
-+
+        """ rflag, chanavg, chanbin=2, propagate flags forth and back """
-+
-+
+        # Make rflag flag something (if no flags are added, the flag cube is not written)
-+
+        res, apriori_flags, final_flags =\
-+
+            self.run_auto_flag_preavg_propagation(chanbin=5, ims=self.vis,
-+
+                                                  mode='rflag', extendflags=False)
-+
-+
+        self.assertEqual(res['total'], 1024)
-+
+        # Before CAS-12727, there is some 'loss' of flags. This would be: 35
-+
+        # Instead of >= 208
-+
+        self.assertEqual(res['flagged'], 236)
-+
+        self.assertTrue(self.check_flags_preserved(apriori_flags, final_flags),
-+
+                        'Not all the flags set "before" are set "after"')
-+
-+
+    def test_propagation_clip_chanbin_64(self):
-+
+        """ clip, chanavg, chanbin=64 (all), propagate flags forth and back """
-+
-+
+        # Make clip flag something (if no flags are added, the flag cube is not written)
-+
+        # Use min=0.0025 to flag very little but still something
-+
+        res, apriori_flags, final_flags =\
-+
+            self.run_auto_flag_preavg_propagation(chanbin=64, ims=self.vis,
-+
+                                                  clipminmax=[0.004, 0.1])
-+
-+
+        self.assertEqual(res['total'], 1024)
-+
+        # Before CAS-12727, there is some 'loss' of flags. This would be: 196
-+
+        # (the total is >= 16 a priori, but losing some of the initial flags which would
-+
+        # be overwritten as False).
-+
+        self.assertEqual(res['flagged'], 205)
-+
+        self.assertTrue(self.check_flags_preserved(apriori_flags, final_flags),
-+
+                        'Not all the flags set "before" are set "after"')
-+
-+
-+
+@unittest.skipIf(False,
-+
+                 "These tests were added in CAS-12737. Not clear what would be the right"
-+
+                 "place for them.")
-+
+class test_flags_propagation_timeavg(test_flags_propagation_base):
-+
+    """
-+
+    Tests on the number and positions of flags when using
-+
+       flagdata + timeavg + autoflag_methods AND the dataset is already flagged
-+
+    ... where timeavg is implemented via the AveragingTVI
-+
+    This is to make sure that flags set before the flagdata command are preserved
-+
+    (CAS-12737). Similarly as in the tests test_flags_propagation_timeavg, the tests of this
-+
+    class check the expected number of flags from several methods (clip, tfcrop, rflag) and
-+
+    that all the data points originally flagged are still flagged after applying, in the
-+
+    exact same positions.
-+
-+
+    Uses the small VLA dataset from "data4timeavg" which has enough integrations to test
-+
+    a range of timebins
-+
-+
+    To illustrate the potential "loss" of flags before the fix from CAS-12737, the tests
-+
+    use a range of timebin values (2s...100s) with intentionally sparse "a priori" flags
-+
+    like
-+
+    X 0 X 0 X 0 X 0     (with timebin=2 could produce a total loss of "a priori" flags)
-+
+    or
-+
+    X 0 0 X 0 0 X 0 0   (with timebin=3 could produce a total loss of "a priori" flags)
-+
+    ...
-+
-+
+    There are notes in the comments that give the final number of flags that would be seen
-+
+    before the fix from CAS-12737 (much lower, lower than the initial "a priori" flags).
-+
+    """
-+
-+
+    def setUp(self):
-+
+        self.setUp_data4timeavg()
-+
-+
+    def run_auto_flag_preavg_propagation(self, timebin=2, mode='clip', ims='', **kwargs):
-+
+        """
-+
+        Enables time average and prepares a priori flags in a sparse pattern through rows/
-+
+        time. The pattern is then used to test the (back)propagation of flags after
-+
+        time-averaging.
-+
+        One row or timestamp is flagged every 'timebin' where timebin is an integer flagging
-+
+        step.
-+
+        With timebin=2, 50% of timestamps will be flagged (X 0 X 0 X 0...)
-+
+        With timebin-3, 33% of timestamps will be flagged (X 0 0 X 0 0...) and so on.
-+
+        This pattern maximizes the "loss" of flags as seen in CAS-12737.
-+
+        Uses the column 'TIME_CENTROID' (and the time reference from there) to find the list
-+
+        of timestamps to flag.
-+
-+
+        :param timebin: number of timestamps to average (in time) - does not check actual
-+
+                        integrations times
-+
+        :param mode: one auto-flag mode
-+
+        :param ims: input ms name
-+
+        :param kwargs: to pass mode specific parameters, such as rflag thresholds, etc.
-+
+        :return: res+apriori_flags+final_flags. res is the flagdata summary dict from the
-+
+        MS after applying flagging with timeavg. apriori_flags is the FLAG column before
-+
+        applying timeavg+autoflag_method. final_flags is the FLAG column after applying
-+
+        timeavg+autoflag_method.
-+
+        """
-+
-+
+        def get_unique_ms_times(ims):
-+
+            """
-+
+            Get the list of unique time stamps of the MS (from TIME_CENTROID).
-+
-+
+            :param: ims: an MS name
-+
+            :return: list of unique times in the MS. Times as produced by the quanta tool.
-+
+            """
-+
+            try:
-+
+                tbt = table()
-+
+                tbt.open(ims)
-+
+                times = tbt.getcol('TIME_CENTROID')
-+
-+
+                ref = tbt.getcolkeyword('TIME_CENTROID', 'MEASINFO')['Ref']
-+
+            except RuntimeError as exc:
-+
+                pass
-+
+            finally:
-+
+                tbt.close()
-+
-+
+            centroids = np.unique(times)
-+
+            # Produce time records ready for flagdata.
-+
+            # This could be done without measures tool:
-+
+            # times = [qat.time({'unit': 's', 'value': cent, 'refer': ref, # 'UTC'
-+
+            #                    'type': 'epoch'},
-+
+            #                   form=['ymd'], prec=9)[0]
-+
+            #          for cent in centroids]
-+
+            # Or even with plain string formatting:
-+
+            # times = [time.strftime('%Y/%m/%d/%H:%M:%S.%f',
-+
+            #          time.gmtime(cent)) for cent in  centroids]
-+
+            # But better to produce times with the measures tool and using MEASINFO:
-+
+            qat = quanta()
-+
+            met = measures()
-+
+            times = [qat.time(met.epoch('ref', '{}s'.format(cent))['m0'], form=['ymd'],
-+
+                              prec=9)[0]
-+
+                     for cent in centroids]
-+
+            return times
-+
-+
+        flagdata(vis=ims, mode='unflag')
-+
-+
+        # Pre-flag some timestamps, at 'timebin' steps
-+
+        times = get_unique_ms_times(ims)
-+
+        flag_times = ['timerange={}'.format(one) for one in times[0::timebin]]
-+
-+
+        flagdata(vis=ims, mode='list', inpfile=flag_times)
-+
-+
+        apriori_flags = self.get_flags(self.vis)
-+
-+
+        res_avg = flagdata(vis=ims, mode=mode, timeavg=True, timebin='{}s'.
-+
+                           format(timebin), **kwargs)
-+
-+
+        res = flagdata(vis=ims, mode='summary')
-+
-+
+        final_flags = self.get_flags(self.vis)
-+
-+
+        return res, apriori_flags, final_flags
-+
-+
+    def test_propagation_clip_timebin_2s(self):
-+
+        """ clip, timeavg, timebin=2, propagate flags forth and back """
-+
-+
+        # Make clip flag something (if no flags are added, the flag cube is not written)
-+
+        res, apriori_flags, final_flags =\
-+
+            self.run_auto_flag_preavg_propagation(timebin=2, ims=self.vis,
-+
+                                                  clipminmax=[0.0, 0.1])
-+
-+
+        self.assertEqual(res['total'], 25600)
-+
+        # Before CAS-12727, there is some 'loss' of flags. This would be: 1490
-+
+        self.assertEqual(res['flagged'], 10311)
-+
+        self.assertTrue(self.check_flags_preserved(apriori_flags, final_flags),
-+
+                        'Not all the flags set "before" are set "after"')
-+
-+
+    def test_propagation_clip_timebin_5s(self):
-+
+        """ clip, timeavg, timebin=5, propagate flags forth and back """
-+
-+
+        # Make clip flag something (if no flags are added, the flag cube is not written)
-+
+        res, apriori_flags, final_flags =\
-+
+            self.run_auto_flag_preavg_propagation(timebin=5, ims=self.vis,
-+
+                                                  clipminmax=[0.001, 0.1])
-+
-+
+        self.assertEqual(res['total'], 25600)
-+
+        # Before CAS-12727, there is some 'loss' of flags. This would be: 1339
-+
+        self.assertEqual(res['flagged'], 5140)
-+
+        self.assertTrue(self.check_flags_preserved(apriori_flags, final_flags),
-+
+                        'Not all the flags set "before" are set "after"')
-+
-+
+    def test_propagation_tfcrop_timebin_20(self):
-+
+        """ tfcrop, timeavg, timebin=20, propagate flags forth and back """
-+
-+
+        res, apriori_flags, final_flags =\
-+
+            self.run_auto_flag_preavg_propagation(timebin=20, ims=self.vis,
-+
+                                                  mode='tfcrop', extendflags=False)
-+
-+
+        self.assertEqual(res['total'], 25600)
-+
+        # Before CAS-12727, there is some 'loss' of flags. This would be: 68
-+
+        # Instead of >= 1280  (= 5 rows x 4 pol x 64 chan)
-+
+        self.assertEqual(res['flagged'], 2905)
-+
+        self.assertTrue(self.check_flags_preserved(apriori_flags, final_flags),
-+
+                        'Not all the flags set "before" are set "after"')
-+
-+
+    def test_propagation_rflag_timebin_20(self):
-+
+        """ rflag, timeavg, timebin=20, propagate flags forth and back """
-+
-+
+        res, apriori_flags, final_flags =\
-+
+            self.run_auto_flag_preavg_propagation(timebin=20, ims=self.vis,
-+
+                                                  mode='rflag', extendflags=False)
-+
-+
+        self.assertEqual(res['total'], 25600)
-+
+        # Before CAS-12727, there is some 'loss' of flags. This would be: 756
-+
+        # Instead of >= 1280
-+
+        self.assertEqual(res['flagged'], 3586)
-+
+        self.assertTrue(self.check_flags_preserved(apriori_flags, final_flags),
-+
+                        'Not all the flags set "before" are set "after"')
-+
-+
+    def test_propagation_clip_timebin_100s(self):
-+
+        """ clip, timeavg, timebin=100, propagate flags forth and back """
-+
-+
+        # Make clip flag something (if no flags are added, the flag cube is not written)
-+
+        res, apriori_flags, final_flags =\
-+
+            self.run_auto_flag_preavg_propagation(timebin=100, ims=self.vis,
-+
+                                                  clipminmax=[0.001, 0.1])
-+
-+
+        self.assertEqual(res['total'], 25600)
-+
+        # Before CAS-12727, there is some 'loss' of flags. This would be: 1339
-+
+        self.assertEqual(res['flagged'], 1609)
-+
+        self.assertTrue(self.check_flags_preserved(apriori_flags, final_flags),
-+
+                        'Not all the flags set "before" are set "after"')
-+
-+
-4780
+class test_forbid_avg_in_non_autoflagging_list(test_base):
-4781
+    """
-4782
+    CAS-12294: forbid the use of timeavg or chanavg in methods other than the
-4783
+    auto-flagging methods (clip, tfcrop, rflag) when given inside the list of
-4784
+    commands in list mode.
-4785
+    """
-4786
-4787
+    def setUp(self):
-4788
+        self.setUp_data4tfcrop()
-4789
-5089
+            test_tsys,
-5090
+            test_bandpass,
-5091
+            test_newcal,
-5092
+            test_weight_spectrum,
-5093
+            test_float_column,
-5094
+            test_tbuff,
-5095
+            TestMergeManualTimerange,
-5096
+            test_preaveraging,
-5097
+            test_preaveraging_rflag_residual,
-5098
+            test_virtual_col,
-+
+            test_flags_propagation_channelavg,
-+
+            test_flags_propagation_timeavg,
-5101
+            test_forbid_avg_in_non_autoflagging_list,
-5102
+            test_list_modes_forbidden_with_avg,
-5103
+            test_auto_methods_display,
-5104
+            cleanup]
-5105
-5106
+if is_CASA6:
-5107
+    if __name__ == '__main__':
-5108
+        unittest.main()

casa6

Commits

Jira Issues

Add shortcut