Commits

Sandra M Castro authored 37724b7d780
Copied test_importasdm_mms from casa5 to casa6. Made both tests compliant with Python 3 and

identical to each other. The tests handle when mpi is not available in the system and runs in serial in both linux and macOS.

casa5/gcwrap/python/scripts/tests/test_importasdm_mms.py

Modified
1 1 #############################################################################
2 2 # $Id:$
3 3 # Test Name: #
4 4 # Regression Test Script for ASDM version 1.2, 1.3 import to MS #
5 -# and the "inverse filler" task exportasdm
5 +# and the "inverse filler" task exportasdm
6 6 # #
7 7 # Rationale for Inclusion: #
8 8 # The conversion of ASDM to MS and back needs to be verified. #
9 -# #
9 +# #
10 10 # Features tested: #
11 11 # 1) Is the import performed without raising exceptions #
12 12 # 2) Do all expected tables exist #
13 13 # 3) Can the MS be opened #
14 14 # 4) Do the tables contain expected values #
15 15 # 5) Is exportasdm performed without raising exceptions #
16 16 # 6) Is the created ASDM well-formed (XML) and complete #
17 17 # 7) Can the resulting ASDM be reimported without raising exceptions #
18 18 # 8) Does it have the same number of integrations as the original #
19 19 # 9) Can a mixed mode ASDM be imported in lazy mode #
20 20 # #
21 21 # Input data: #
22 22 # one dataset for the filler of ASDM 1.0 #
23 23 # one simulated MS dataset #
24 24 # #
25 25 #############################################################################
26 26 import os
27 27 import sys
28 28 import shutil
29 -from __main__ import default
30 -from tasks import importasdm, flagdata, exportasdm, listpartition
31 -from taskinit import mstool, tbtool, msmdtool
32 -#import testhelper as th
33 -from casatestutils import testhelper as th
34 29 import unittest
35 -import partitionhelper as ph
36 -from parallel.parallel_data_helper import ParallelDataHelper
37 30
31 +is_CASA6 = False
32 +has_mpi = False
33 +try:
34 + from casatools import ctsys, ms, table, msmetadata
35 + from casatasks import importasdm, flagdata, listpartition, exportasdm
36 + from casatasks.private import partitionhelper as ph
37 + from casatasks.private.parallel.parallel_data_helper import ParallelDataHelper
38 + is_CASA6 = True
39 + # make local copies of the tools
40 +# tblocal = table( )
41 +# mslocal = ms( )
42 +# mdlocal = msmetadata()
43 + mdlocal = msmetadata()
44 + mslocal = ms()
45 + tblocal = table()
46 +
47 +
48 + ctsys_resolve = ctsys.resolve
49 + # CASAtasks does not use default
50 + def default(atask):
51 + pass
52 +except ImportError:
53 + from __main__ import default
54 + from tasks import importasdm, flagdata, exportasdm, listpartition
55 + from taskinit import mstool, tbtool, msmdtool
56 + import partitionhelper as ph
57 + from parallel.parallel_data_helper import ParallelDataHelper
58 + from mpi4casa.MPICommandClient import MPICommandClient
59 + from mpi4casa.MPIEnvironment import MPIEnvironment
60 + has_mpi = True
61 +
62 + mslocal = mstool()
63 + mdlocal = msmdtool()
64 + tblocal = tbtool()
65 +
66 +# mst = mstool
67 +# tbt = tbtool
68 +# mdt = msmdtool
69 + # make local copies of the tools
70 +# tblocal = tbtool()
71 +# mslocal = mstool()
72 +# mdlocal = mdt()
73 +
74 + def ctsys_resolve(apath):
75 + dataPath = os.path.join(os.environ.get('CASAPATH').split()[0],'casatestdata/')
76 + return os.path.join(dataPath,apath)
77 +
78 +# Safeguarding when running the test in macOS
79 +if is_CASA6:
80 + try:
81 + from casampi.MPIEnvironment import MPIEnvironment
82 + from casampi.MPICommandClient import MPICommandClient
83 + has_mpi = True
84 + except:
85 + casalog.post('casampi is not available. Running in serial', 'WARN')
86 +
87 +
88 +from casatestutils import testhelper as th
38 89 myname = 'test_importasdm'
39 90
40 91 # default ASDM dataset name
41 92 myasdm_dataset_name = 'uid___X5f_X18951_X1'
42 93 myms_dataset_name = 'M51.ms'
43 94
44 95 # name of the resulting MS
45 96 msname = myasdm_dataset_name+'.ms'
46 97
47 98 # name of the exported ASDM
48 99 asdmname = myms_dataset_name+'.asdm'
49 100
50 101 # name of the reimported MS
51 102 reimp_msname = 'reimported-'+myms_dataset_name
52 103
53 104 # Path to input data
54 -datapath=os.environ.get('CASAPATH').split()[0]+'/casatestdata/unittest/importasdm/'
105 +datapath = ctsys_resolve('unittest/importasdm/')
106 +
55 107
56 108 # make local copies of the tools
57 -tblocal = tbtool()
58 -mslocal = mstool()
109 +#tblocal = tblocal()
110 +#mslocal = mslocal()
111 +#mdlocal = mdlocal()
59 112
60 113 def checktable(thename, theexpectation):
61 114 global msname, myname
62 115 tblocal.open(msname+"/"+thename)
63 116 if thename == "":
64 117 thename = "MAIN"
65 118 for mycell in theexpectation:
66 - print myname, ": comparing ", mycell
119 + print(myname, ": comparing ", mycell)
67 120 value = tblocal.getcell(mycell[0], mycell[1])
68 121 # see if value is array
69 122 try:
70 123 isarray = value.__len__
71 124 except:
72 125 # it's not an array
73 126 # zero tolerance?
74 127 if mycell[3] == 0:
75 128 in_agreement = (value == mycell[2])
76 129 else:
77 - in_agreement = ( abs(value - mycell[2]) < mycell[3])
130 + in_agreement = ( abs(value - mycell[2]) < mycell[3])
78 131 else:
79 132 # it's an array
80 133 # zero tolerance?
81 134 if mycell[3] == 0:
82 - in_agreement = (value == mycell[2]).all()
135 + in_agreement = (value == mycell[2]).all()
83 136 else:
84 - in_agreement = (abs(value - mycell[2]) < mycell[3]).all()
137 + in_agreement = (abs(value - mycell[2]) < mycell[3]).all()
85 138 if not in_agreement:
86 - print myname, ": Error in MS subtable", thename, ":"
87 - print " column ", mycell[0], " row ", mycell[1], " contains ", value
88 - print " expected value is ", mycell[2]
139 + print(myname, ": Error in MS subtable", thename, ":")
140 + print(" column ", mycell[0], " row ", mycell[1], " contains ", value)
141 + print(" expected value is ", mycell[2])
89 142 tblocal.close()
90 143 return False
91 144 tblocal.close()
92 - print myname, ": table ", thename, " as expected."
145 + print(myname, ": table ", thename, " as expected.")
93 146 return True
94 147
95 -#########################
96 148
97 149 def verify_asdm(asdmname, withPointing):
98 - print "Verifying asdm ", asdmname
150 + print("Verifying asdm ", asdmname)
99 151 if(not os.path.exists(asdmname)):
100 - print "asdm ", asdmname, " doesn't exist."
152 + print("asdm ", asdmname, " doesn't exist.")
101 153 raise Exception
102 154 # test for the existence of all obligatory tables
103 155 allTables = [ "Antenna.xml",
104 156 "ASDM.xml",
105 157 # "CalData.xml",
106 158 # "CalDelay.xml",
107 159 # "CalReduction.xml",
108 160 "ConfigDescription.xml",
109 161 "CorrelatorMode.xml",
110 162 "DataDescription.xml",
124 176 "SpectralWindow.xml",
125 177 "State.xml",
126 178 "Station.xml",
127 179 "Subscan.xml",
128 180 "SwitchCycle.xml"
129 181 ]
130 182 isOK = True
131 183 for fileName in allTables:
132 184 filePath = asdmname+'/'+fileName
133 185 if(not os.path.exists(filePath)):
134 - print "ASDM table file ", filePath, " doesn't exist."
186 + print("ASDM table file ", filePath, " doesn't exist.")
135 187 isOK = False
136 188 else:
137 189 # test if well formed
138 190 rval = os.system('xmllint --noout '+filePath)
139 191 if(rval !=0):
140 - print "Table ", filePath, " is not a well formed XML document."
192 + print("Table ", filePath, " is not a well formed XML document.")
141 193 isOK = False
142 194
143 - print "Note: xml validation not possible since ASDM DTDs (schemas) not yet online."
144 -
195 + print("Note: xml validation not possible since ASDM DTDs (schemas) not yet online.")
196 +
145 197 if(not os.path.exists(asdmname+"/ASDMBinary")):
146 - print "ASDM binary directory "+asdmname+"/ASDMBinary doesn't exist."
198 + print("ASDM binary directory "+asdmname+"/ASDMBinary doesn't exist.")
147 199 isOK = False
148 200
149 201 if(withPointing and not os.path.exists(asdmname+"/Pointing.bin")):
150 - print "ASDM binary file "+asdmname+"/Pointing.bin doesn't exist."
202 + print("ASDM binary file "+asdmname+"/Pointing.bin doesn't exist.")
151 203 isOK = False
152 204
153 205 if (not isOK):
154 206 raise Exception
155 207
156 208 # Setup for different data importing
157 209 class test_base(unittest.TestCase):
158 210 def setUp_m51(self):
159 211 res = None
160 212 if(os.path.exists(myasdm_dataset_name)):
161 213 shutil.rmtree(myasdm_dataset_name)
162 214
163 215 shutil.copytree(datapath + myasdm_dataset_name, myasdm_dataset_name)
164 216 shutil.copytree(datapath + myms_dataset_name, myms_dataset_name)
165 217 default(importasdm)
166 218
167 219 def setUp_xosro(self):
168 220 self.asdm = 'X_osro_013.55979.93803716435'
169 221 if(not os.path.lexists(self.asdm)):
170 222 os.system('ln -s '+datapath+self.asdm +' '+self.asdm)
171 -
223 +
172 224 default(importasdm)
173 225
174 226
175 227 def setUp_autocorr(self):
176 228 self.asdm = 'AutocorrASDM'
177 229 if(not os.path.lexists(self.asdm)):
178 230 os.system('ln -s '+datapath+self.asdm +' '+self.asdm)
179 -
231 +
180 232 default(importasdm)
181 233
182 234 def setUp_acaex(self):
183 235 res = None
184 236 myasdmname = 'uid___A002_X72bc38_X000' # ACA example ASDM with mixed pol/channelisation
185 237
186 238 os.system('ln -sf '+datapath+myasdmname)
187 239 default(importasdm)
188 240
189 241 def setUp_12mex(self):
203 255 def setUp_flags(self):
204 256 res = None
205 257 myasdmname = 'test_uid___A002_X997a62_X8c-short' # Flag.xml is modified
206 258
207 259 os.system('ln -sf '+datapath+myasdmname)
208 260 default(importasdm)
209 261
210 262 def setUp_SD(self):
211 263 res = None
212 264 # Single-dish ASDM, 1 scan, 9 spws, 4 antennas, 10 baselines, 4 auto-corrs + cross
213 - myasdmname = 'uid___A002_X6218fb_X264'
265 + myasdmname = 'uid___A002_X6218fb_X264'
214 266
215 267 os.system('ln -sf '+datapath+myasdmname)
216 268 default(importasdm)
217 269
218 270
219 271 ###########################
220 -# beginning of actual test
221 -
272 +# beginning of actual test
273 +
222 274 class asdm_import8(test_base):
223 275 '''Test importasdm with Multi-MS'''
224 -
276 +
225 277 def setUp(self):
226 278 self.setUp_12mex()
227 279 self.setUp_acaex()
228 -
280 +
229 281 def tearDown(self):
230 282 for myasdmname in ['uid___A002_X71e4ae_X317_short', 'uid___A002_X72bc38_X000']:
231 283 os.system('rm -f '+myasdmname) # a link
232 284 shutil.rmtree(myasdmname+".ms",ignore_errors=True)
233 285 shutil.rmtree(myasdmname+'.ms.flagversions',ignore_errors=True)
234 286 shutil.rmtree("reference.ms",ignore_errors=True)
235 287
236 288 def test_mms1(self):
237 289 '''test_mms1: Create an MMS with default name'''
238 290 myasdmname = 'uid___A002_X71e4ae_X317_short'
239 291 themsname = myasdmname+".ms"
240 292
241 293 importasdm(myasdmname, createmms=True)
242 294 self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
243 -
295 +
244 296 def test_mms2(self):
245 297 '''test_mms2: Create an MMS with default name and lazy=True'''
246 298 myasdmname = 'uid___A002_X71e4ae_X317_short'
247 299 themsname = myasdmname+".ms"
248 300
249 301 importasdm(myasdmname, createmms=True, lazy=True, scans='2')
250 302 self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
251 303
252 304 def test_mms3(self):
253 305 '''test_mms3: Create MMS with separationaxis=spw and lazy=True'''
254 306 myasdmname = 'uid___A002_X71e4ae_X317_short'
255 307 themsname = myasdmname+".ms"
256 308
257 309 importasdm(myasdmname, createmms=True, lazy=True, scans='1,2', separationaxis='spw', flagbackup=False,
258 310 process_flags=False)
259 311 self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
260 312 self.assertEqual(ph.axisType(themsname), 'spw', 'Separation axis of MMS should be spw')
261 -
313 +
262 314 def test_mms4(self):
263 315 '''test_mms4: Create MMS, lazy=True, with separationaxis=scan and scans selection in ASDM'''
264 - retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
316 + retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
265 317
266 318 myasdmname = 'uid___A002_X72bc38_X000'
267 319 themsname = myasdmname + ".ms"
268 320
269 321 # only the first 3 scans to save time
270 322 importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~3', createmms=True, separationaxis='scan',
271 - process_flags=False)
323 + process_flags=False)
272 324 self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
273 325 self.assertEqual(ph.axisType(themsname), 'scan', 'Separation axis of MMS should be scan')
274 - print myname, ": Success! Now checking output ..."
326 + print(myname, ": Success! Now checking output ...")
275 327 mscomponents = set(["ANTENNA/table.dat",
276 328 "DATA_DESCRIPTION/table.dat",
277 329 "FEED/table.dat",
278 330 "FIELD/table.dat",
279 331 "FLAG_CMD/table.dat",
280 332 "HISTORY/table.dat",
281 333 "OBSERVATION/table.dat",
282 334 "POINTING/table.dat",
283 335 "POLARIZATION/table.dat",
284 336 "PROCESSOR/table.dat",
296 348 "POINTING/table.f0",
297 349 "POLARIZATION/table.f0",
298 350 "PROCESSOR/table.f0",
299 351 "SOURCE/table.f0",
300 352 "SPECTRAL_WINDOW/table.f0",
301 353 "STATE/table.f0",
302 354 "SYSCAL/table.f0"
303 355 ])
304 356 for name in mscomponents:
305 357 if not os.access(themsname+"/"+name, os.F_OK):
306 - print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
358 + print(myname, ": Error ", themsname+"/"+name, "doesn't exist ...")
307 359 retValue['success']=False
308 360 retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
309 361 else:
310 - print myname, ": ", name, "present."
311 - print myname, ": MS exists. All tables present. Try opening as MS ..."
362 + print(myname, ": ", name, "present.")
363 + print(myname, ": MS exists. All tables present. Try opening as MS ...")
312 364 try:
313 365 mslocal.open(themsname)
314 366 except:
315 - print myname, ": Error Cannot open MS table", themsname
367 + print(myname, ": Error Cannot open MS table", themsname)
316 368 retValue['success']=False
317 369 retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
318 370 else:
319 371 mslocal.close()
320 - print myname, ": OK. Checking tables in detail ..."
321 -
372 + print(myname, ": OK. Checking tables in detail ...")
373 +
322 374 importasdm(asdm=myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3',
323 375 createmms=True, separationaxis='scan', process_flags=False)
324 376
325 377 if(os.path.exists('reference.ms')):
326 378 retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
327 379 +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
328 380 if not retValue['success']:
329 - print "ERROR: DATA does not agree with reference."
381 + print("ERROR: DATA does not agree with reference.")
330 382 else:
331 - print "DATA columns agree."
383 + print("DATA columns agree.")
332 384 retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
333 385 +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
334 386 if not retValueTmp:
335 - print "ERROR: FLAG does not agree with reference."
387 + print("ERROR: FLAG does not agree with reference.")
336 388 else:
337 - print "FLAG columns agree."
389 + print("FLAG columns agree.")
338 390
339 391 retValue['success'] = retValue['success'] and retValueTmp
340 392
341 393 for subtname in ["ANTENNA",
342 394 "DATA_DESCRIPTION",
343 395 "FEED",
344 396 "FIELD",
345 397 "FLAG_CMD",
346 398 "OBSERVATION",
347 399 "POINTING",
348 400 "POLARIZATION",
349 401 "PROCESSOR",
350 402 "SOURCE",
351 403 "SPECTRAL_WINDOW",
352 404 "STATE",
353 405 "SYSCAL"]:
354 -
355 - print "\n*** Subtable ",subtname
406 +
407 + print("\n*** Subtable ",subtname)
356 408 excllist = []
357 409 if subtname=='SOURCE':
358 410 excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
359 411 if subtname=='SYSCAL':
360 412 excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
361 413 if subtname=='SPECTRAL_WINDOW':
362 414 excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION']
363 - for colname in excllist:
415 + for colname in excllist:
364 416 retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
365 417 themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
366 -
367 - try:
418 +
419 + try:
368 420 retValue['success'] = th.compTables('reference.ms/'+subtname,
369 - themsname+'/'+subtname, excllist,
421 + themsname+'/'+subtname, excllist,
370 422 0.01) and retValue['success']
371 423 except:
372 424 retValue['success'] = False
373 - print "ERROR for table ", subtname
374 -
375 - print retValue
425 + print("ERROR for table ", subtname)
426 +
427 + print(retValue )
376 428 self.assertTrue(retValue['success'],retValue['error_msgs'])
377 -
429 +
378 430 @unittest.skip('Skip until a suitable ASDM is found to run this test')
379 431 def test_mms5(self):
380 432 '''test_mms5: Create 2 MMS, 2 flagversions and 2 online flag files'''
381 433 myasdmname = 'uid___A002_X71e4ae_X317_short'
382 434 themsname = myasdmname+".ms"
383 435 wvrmsname = myasdmname+'-wvr-corrected.ms'
384 436 flagfile1 = myasdmname+'_cmd.txt'
385 437 flagfile2 = myasdmname+'-wvr-corrected'+'_cmd.txt'
386 -
438 +
387 439 importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~4', wvr_corrected_data='both', savecmds=True,
388 440 createmms=True)
389 441 self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
390 442 self.assertTrue(ParallelDataHelper.isParallelMS(wvrmsname), 'Output is not a Multi-MS')
391 443 self.assertTrue(os.path.exists(flagfile1))
392 444 self.assertTrue(os.path.exists(flagfile2))
393 445
394 -
395 446 def test_mms6(self):
396 447 '''test_mms6: Create MMS and lazy=True'''
397 - retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
448 + retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
398 449 myasdmname = 'uid___A002_X71e4ae_X317_short'
399 450 themsname = myasdmname+".ms"
400 451 flagfile = myasdmname+'_cmd.txt'
401 452
402 453 self.res = importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~4', createmms=True,
403 454 savecmds=True) # only the first 4 scans to save time
404 455 self.assertEqual(self.res, None)
405 456 self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
406 457 self.assertTrue(os.path.exists(flagfile))
407 - print myname, ": Success! Now checking output ..."
458 + print(myname, ": Success! Now checking output ...")
408 459 mscomponents = set(["ANTENNA/table.dat",
409 460 "DATA_DESCRIPTION/table.dat",
410 461 "FEED/table.dat",
411 462 "FIELD/table.dat",
412 463 "FLAG_CMD/table.dat",
413 464 "HISTORY/table.dat",
414 465 "OBSERVATION/table.dat",
415 466 "POINTING/table.dat",
416 467 "POLARIZATION/table.dat",
417 468 "PROCESSOR/table.dat",
429 480 "POINTING/table.f0",
430 481 "POLARIZATION/table.f0",
431 482 "PROCESSOR/table.f0",
432 483 "SOURCE/table.f0",
433 484 "SPECTRAL_WINDOW/table.f0",
434 485 "STATE/table.f0",
435 486 "SYSCAL/table.f0"
436 487 ])
437 488 for name in mscomponents:
438 489 if not os.access(themsname+"/"+name, os.F_OK):
439 - print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
490 + print(myname, ": Error ", themsname+"/"+name, "doesn't exist ...")
440 491 retValue['success']=False
441 492 retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
442 493 else:
443 - print myname, ": ", name, "present."
444 - print myname, ": MS exists. All tables present. Try opening as MS ..."
494 + print (myname, ": ", name, "present.")
495 + print(myname, ": MS exists. All tables present. Try opening as MS ...")
445 496 try:
446 497 mslocal.open(themsname)
447 498 mslocal.close()
448 - print myname, ": MS can be opened. Now testing the changing of the asdmref ..."
499 + print(myname, ": MS can be opened. Now testing the changing of the asdmref ...")
449 500 mslocal.open(themsname)
450 501 mslocal.asdmref("./moved_"+myasdmname)
451 502 mslocal.close()
452 503 os.system("mv "+myasdmname+" moved_"+myasdmname)
453 -
504 +
454 505 mslocal.open(themsname)
455 -
506 +
456 507 except:
457 - print myname, ": Error Cannot open MS table", themsname
508 + print(myname, ": Error Cannot open MS table", themsname)
458 509 retValue['success']=False
459 510 retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
460 511 else:
461 512 mslocal.close()
462 - print myname, ": OK. Checking tables in detail ..."
463 -
513 + print(myname, ": OK. Checking tables in detail ...")
514 +
464 515 importasdm(asdm="moved_"+myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3', createmms=True)
465 516
466 517 if(os.path.exists('reference.ms')):
467 518 retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
468 519 +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
469 520 if not retValue['success']:
470 - print "ERROR: DATA does not agree with reference."
521 + print("ERROR: DATA does not agree with reference.")
471 522 else:
472 - print "DATA columns agree."
523 + print("DATA columns agree.")
473 524
474 525 retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
475 526 +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.WEIGHT,t2.WEIGHT, 1.e-06)))") == 0
476 527 if not retValueTmp:
477 - print "ERROR: WEIGHT does not agree with reference."
528 + print("ERROR: WEIGHT does not agree with reference.")
478 529 else:
479 - print "WEIGHT columns agree."
480 -
530 + print("WEIGHT columns agree.")
531 +
481 532 retValueTmp2 = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
482 533 +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
483 534 if not retValueTmp2:
484 - print "ERROR: FLAG does not agree with reference."
535 + print("ERROR: FLAG does not agree with reference.")
485 536 else:
486 - print "FLAG columns agree."
537 + print("FLAG columns agree.")
487 538
488 539 retValue['success'] = retValue['success'] and retValueTmp and retValueTmp2
489 540
490 541 for subtname in ["ANTENNA",
491 542 "DATA_DESCRIPTION",
492 543 "FEED",
493 544 "FIELD",
494 545 "FLAG_CMD",
495 546 "OBSERVATION",
496 547 "POLARIZATION",
497 548 "PROCESSOR",
498 549 "SOURCE",
499 550 "SPECTRAL_WINDOW",
500 551 "STATE",
501 552 "SYSCAL"]:
502 -
503 - print "\n*** Subtable ",subtname
553 +
554 + print("\n*** Subtable ",subtname)
504 555 excllist = []
505 556 if subtname=='SOURCE':
506 557 excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
507 558 if subtname=='SYSCAL':
508 559 excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
509 560 if subtname=='SPECTRAL_WINDOW':
510 561 excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE']
511 562 for colname in excllist:
512 563 if colname!='ASSOC_NATURE':
513 564 retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
514 565 themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
515 566 if subtname=='POLARIZATION':
516 567 excllist=['CORR_TYPE', 'CORR_PRODUCT']
517 - for colname in excllist:
568 + for colname in excllist:
518 569 retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION',
519 570 themsname+'/POLARIZATION', colname, 0.01) and retValue['success']
520 - try:
571 + try:
521 572 retValue['success'] = th.compTables('reference.ms/'+subtname,
522 - themsname+'/'+subtname, excllist,
573 + themsname+'/'+subtname, excllist,
523 574 0.01) and retValue['success']
524 575 except:
525 576 retValue['success'] = False
526 - print "ERROR for table ", subtname
577 + print("ERROR for table ", subtname)
527 578
528 579 os.system("mv moved_"+myasdmname+" "+myasdmname)
529 -
580 +
530 581 self.assertTrue(retValue['success'],retValue['error_msgs'])
531 -
582 +
532 583 def test_mms7(self):
533 584 '''Asdm-import: Test good 12 m ASDM with mixed pol/channelisation input with default filler in lazy mode with reading the BDF flags. Output MMS'''
534 - retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
585 + retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
535 586
536 587 myasdmname = 'uid___A002_X71e4ae_X317_short'
537 588 themsname = myasdmname+".ms"
538 589
539 - self.res = importasdm(myasdmname, vis=themsname, lazy=True, bdfflags=True, createmms=True)
590 + self.res = importasdm(myasdmname, vis=themsname, lazy=True, bdfflags=True, createmms=True)
540 591 self.assertEqual(self.res, None)
541 - print myname, ": Success! Now checking output ..."
592 + print(myname, ": Success! Now checking output ...")
542 593 mscomponents = set(["ANTENNA/table.dat",
543 594 "DATA_DESCRIPTION/table.dat",
544 595 "FEED/table.dat",
545 596 "FIELD/table.dat",
546 597 "FLAG_CMD/table.dat",
547 598 "HISTORY/table.dat",
548 599 "OBSERVATION/table.dat",
549 600 "POINTING/table.dat",
550 601 "POLARIZATION/table.dat",
551 602 "PROCESSOR/table.dat",
563 614 "POINTING/table.f0",
564 615 "POLARIZATION/table.f0",
565 616 "PROCESSOR/table.f0",
566 617 "SOURCE/table.f0",
567 618 "SPECTRAL_WINDOW/table.f0",
568 619 "STATE/table.f0",
569 620 "SYSCAL/table.f0"
570 621 ])
571 622 for name in mscomponents:
572 623 if not os.access(themsname+"/"+name, os.F_OK):
573 - print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
624 + print(myname, ": Error ", themsname+"/"+name, "doesn't exist ...")
574 625 retValue['success']=False
575 626 retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
576 627 else:
577 - print myname, ": ", name, "present."
578 - print myname, ": MS exists. All tables present. Try opening as MS ..."
628 + print(myname, ": ", name, "present.")
629 + print(myname, ": MS exists. All tables present. Try opening as MS ...")
579 630 try:
580 631 mslocal.open(themsname)
581 632 except:
582 - print myname, ": Error Cannot open MS table", themsname
633 + print(myname, ": Error Cannot open MS table", themsname)
583 634 retValue['success']=False
584 635 retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
585 636 else:
586 637 mslocal.close()
587 - print myname, ": OK. Checking tables in detail ..."
588 -
638 + print(myname, ": OK. Checking tables in detail ...")
639 +
589 640 importasdm(asdm=myasdmname, vis='reference.ms', lazy=True, overwrite=True, bdfflags=False, createmms=True)
590 641
591 642 if(os.path.exists('reference.ms')):
592 643 retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
593 644 +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
594 645 if not retValue['success']:
595 - print "ERROR: DATA does not agree with reference."
646 + print("ERROR: DATA does not agree with reference.")
596 647 else:
597 - print "DATA columns agree."
648 + print("DATA columns agree.")
598 649 retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
599 650 +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") != 0
600 651 if not retValueTmp:
601 - print "ERROR: FLAG columns do agree with reference but they shouldn't."
652 + print("ERROR: FLAG columns do agree with reference but they shouldn't.")
602 653 else:
603 - print "FLAG columns do not agree as expected."
654 + print("FLAG columns do not agree as expected.")
604 655
605 656 retValue['success'] = retValue['success'] and retValueTmp
606 657
607 658 for subtname in ["ANTENNA",
608 659 "DATA_DESCRIPTION",
609 660 "FEED",
610 661 "FIELD",
611 662 "FLAG_CMD",
612 663 "OBSERVATION",
613 664 "POLARIZATION",
614 665 "PROCESSOR",
615 666 "SOURCE",
616 667 "SPECTRAL_WINDOW",
617 668 "STATE",
618 669 "SYSCAL"]:
619 -
620 - print "\n*** Subtable ",subtname
670 +
671 + print("\n*** Subtable ",subtname)
621 672 excllist = []
622 673 if subtname=='SOURCE':
623 674 excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
624 675 if subtname=='SYSCAL':
625 676 excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
626 677 if subtname=='SPECTRAL_WINDOW':
627 678 excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE']
628 679 for colname in excllist:
629 680 if colname!='ASSOC_NATURE':
630 681 retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
631 682 themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
632 683 if subtname=='POLARIZATION':
633 684 excllist=['CORR_TYPE', 'CORR_PRODUCT']
634 - for colname in excllist:
685 + for colname in excllist:
635 686 retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION',
636 687 themsname+'/POLARIZATION', colname, 0.01) and retValue['success']
637 - try:
688 + try:
638 689 retValue['success'] = th.compTables('reference.ms/'+subtname,
639 - themsname+'/'+subtname, excllist,
690 + themsname+'/'+subtname, excllist,
640 691 0.01) and retValue['success']
641 692 except:
642 693 retValue['success'] = False
643 - print "ERROR for table ", subtname
644 -
645 -
694 + print("ERROR for table ", subtname)
695 +
696 +
646 697 self.assertTrue(retValue['success'],retValue['error_msgs'])
647 -
648 698
649 699 class asdm_import9(test_base):
650 700 '''Test importasdm with spw selection in online flags'''
651 -
701 +
652 702 def setUp(self):
653 703 self.setUp_flags()
654 -
704 +
655 705 def tearDown(self):
656 706 for myasdmname in ['test_uid___A002_X997a62_X8c-short']:
657 707 os.system('rm -f '+myasdmname) # a link
658 708 shutil.rmtree(myasdmname+".ms",ignore_errors=True)
659 709 shutil.rmtree(myasdmname+'.ms.flagversions',ignore_errors=True)
660 710
661 711 def test_online1(self):
662 712 '''importasdm: online flags file with spw selection by name'''
663 713 myasdmname = 'test_uid___A002_X997a62_X8c-short'
664 714 themsname = myasdmname+".ms"
665 715 flagfile = myasdmname+'_cmd.txt'
666 716
667 717 importasdm(myasdmname, vis=themsname, scans='1', savecmds=True)
668 - flist = open(flagfile,'r').read().splitlines()
718 + with open(flagfile) as f:
719 + flist = f.readlines()
669 720 self.assertTrue(flist[0].__contains__('spw'))
670 721 self.assertFalse(flist[3].__contains__('WVR#Antenna'))
671 722 self.assertFalse(flist[4].__contains__('WVR#Antenna'))
672 723
673 -
674 724 class asdm_import10(test_base):
675 725 '''Test importasdm with single-dish data'''
676 -
726 +
677 727 def setUp(self):
678 728 self.setUp_SD()
679 -
729 +
680 730 def tearDown(self):
681 731 for myasdmname in ['uid___A002_X6218fb_X264']:
682 732 # os.system('rm -f '+myasdmname) # a link
683 733 os.unlink('uid___A002_X6218fb_X264')
684 734 shutil.rmtree(myasdmname+".ms",ignore_errors=True)
685 735 shutil.rmtree(myasdmname+'.ms.flagversions',ignore_errors=True)
686 736
687 737 def test_float_data_mms(self):
688 738 '''importasdm: Create an MMS from a FLOAT_DATA MS '''
689 739 myasdmname = 'uid___A002_X6218fb_X264'
690 740 themsname = myasdmname+".ms"
691 741
692 742 # The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
693 743 importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1')
694 - self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
744 + self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
695 745 self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
696 746
697 747 def test_sd_data_mms(self):
698 748 '''importasdm: Create an MMS from a single-dish MS and DATA column '''
699 749 myasdmname = 'uid___A002_X6218fb_X264'
700 750 themsname = myasdmname+".ms"
701 751
702 752 importasdm(myasdmname, vis=themsname, scans='1,4', createmms=True, separationaxis='scan', numsubms=2, flagbackup=False)
703 753 self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
704 754 self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)
705 755
706 756 def test_sd_data_mms_baseline_all(self):
707 757 '''importasdm: Create an MMS separated per baseline, using default numsubms '''
708 758 myasdmname = 'uid___A002_X6218fb_X264'
709 759 themsname = myasdmname+".ms"
710 760
711 761 # Create a single-dish MMS with auto and cross correlations
712 762 importasdm(myasdmname, vis=themsname, ocorr_mode='ca', createmms=True, scans='1', separationaxis='baseline')
713 - self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
763 + self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
714 764 self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)
715 -
716 - md = msmdtool()
717 - md.open(themsname)
718 - bsl = md.baselines()
719 - md.close()
720 -
765 +
766 +# md = msmdtool
767 + mdlocal.open(themsname)
768 + bsl = mdlocal.baselines()
769 + mdlocal.close()
770 +
721 771 # Check if all baselines are in there
722 772 self.assertTrue(bsl.all(), 'Not all baselines are in the MMS')
723 773
724 774 def test_float_data_mms_baseline_auto(self):
725 775 '''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline '''
726 776 myasdmname = 'uid___A002_X6218fb_X264'
727 777 themsname = myasdmname+".ms"
728 778
729 779 # The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
730 780 importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1', separationaxis='baseline', numsubms=4)
731 - self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
781 + self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
732 782 self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
733 -
734 - md = msmdtool()
735 - md.open(themsname)
736 - bsl = md.baselines()
737 - md.close()
738 -
783 +
784 +# md = msmdtool
785 + mdlocal.open(themsname)
786 + bsl = mdlocal.baselines()
787 + mdlocal.close()
788 +
739 789 #diagnoals give the auto-corrs
740 790 ac = bsl.diagonal()
741 791 self.assertTrue(ac.all(), 'Not all auto-correlations are there')
742 -
792 +
743 793 # Take the dictionary and compare with original MS
744 794 thisdict = listpartition(vis=themsname, createdict=True)
745 795 self.assertEqual(len(thisdict.keys()), 4, 'There should be 4 subMSs in output MMS')
746 796
747 -
748 797 class asdm_import11(test_base):
749 798 '''Test importasdm with MMS and ephemerides data'''
750 -
799 +
751 800 def setUp(self):
752 801 self.setUp_eph()
753 -
802 +
754 803 def tearDown(self):
755 804 pass
756 805 # for myasdmname in ['uid___A002_X997a62_X8c-short']:
757 806 ## os.system('rm -f '+myasdmname) # a link
758 807 # shutil.rmtree(myasdmname+".ms",ignore_errors=True)
759 808 # shutil.rmtree(myasdmname+'.ms.flagversions',ignore_errors=True)
760 -
809 +
761 810 def test_mms_ephem(self):
762 811 '''Asdm-import: Test good 12 m ASDM with Ephemeris table in lazy mode and MMS'''
763 - retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
812 + retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
764 813
765 814 myasdmname = 'uid___A002_X997a62_X8c-short'
766 815 themsname = myasdmname+".ms"
767 816
768 - self.res = importasdm(myasdmname, vis=themsname, lazy=True, convert_ephem2geo=True,
769 - process_pointing=False, flagbackup=False, createmms=True)
817 + self.res = importasdm(myasdmname, vis=themsname, lazy=True, convert_ephem2geo=True,
818 + process_pointing=False, flagbackup=False, createmms=True)
770 819 self.assertEqual(self.res, None)
771 - print myname, ": Success! Now checking output ..."
820 + print(myname, ": Success! Now checking output ...")
772 821 mscomponents = set(["FIELD/table.dat",
773 822 "FIELD/EPHEM0_Mars_57034.9.tab",
774 823 "FIELD/EPHEM1_Titania_57034.9.tab"
775 824 ])
776 825 for name in mscomponents:
777 826 if not os.access(themsname+"/"+name, os.F_OK):
778 - print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
827 + print(myname, ": Error ", themsname+"/"+name, "doesn't exist ...")
779 828 retValue['success']=False
780 829 retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
781 830 else:
782 - print myname, ": ", name, "present."
783 - print myname, ": MS exists. All relevant tables present. Try opening as MS ..."
831 + print(myname, ": ", name, "present.")
832 + print(myname, ": MS exists. All relevant tables present. Try opening as MS ...")
784 833 try:
785 834 mslocal.open(themsname)
786 835 except:
787 - print myname, ": Error Cannot open MS table", themsname
836 + print(myname, ": Error Cannot open MS table", themsname)
788 837 retValue['success']=False
789 838 retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
790 839 else:
791 840 mslocal.close()
792 - print myname, ": OK."
841 + print(myname, ": OK.")
793 842
794 843 for name in ["FIELD/EPHEM0_Mars_57034.9.tab",
795 844 "FIELD/EPHEM1_Titania_57034.9.tab"]:
796 845 tblocal.open(themsname+"/"+name)
797 846 kw = tblocal.getkeywords()
798 847 tblocal.close()
799 848 geodist = kw['GeoDist'] # (km above reference ellipsoid)
800 849 geolat = kw['GeoLat'] # (deg)
801 850 geolong = kw['GeoLong'] # (deg)
802 - print myname, ": Testing if ephemeris ", name, " was converted to GEO ..."
851 + print(myname, ": Testing if ephemeris ", name, " was converted to GEO ...")
803 852 if not (geodist==geolat==geolong==0.):
804 - print myname, ": ERROR."
853 + print(myname, ": ERROR.")
805 854 retValue['success']=False
806 855 retValue['error_msgs']=retValue['error_msgs']+' Ephemeris was not converted to GEO for '+themsname+'\n'
807 856 else:
808 - print myname, ": OK."
857 + print(myname, ": OK.")
809 858 prsys = kw['posrefsys']
810 - print myname, ": Testing if posrefsys was set correctly ..."
859 + print(myname, ": Testing if posrefsys was set correctly ...")
811 860 if not (prsys=="ICRF/ICRS"):
812 - print myname, ": ERROR."
861 + print(myname, ": ERROR.")
813 862 retValue['success']=False
814 863 retValue['error_msgs']=retValue['error_msgs']+' posrefsys keyword is not ICRF/ICRS '+themsname+'\n'
815 864 else:
816 - print myname, ": OK."
817 -
865 + print(myname, ": OK.")
866 +
818 867
819 868 self.assertTrue(retValue['success'],retValue['error_msgs'])
820 -
869 +
821 870 def suite():
822 871 return [asdm_import8,
823 872 asdm_import9,
824 873 asdm_import10,
825 - asdm_import11]
826 -
827 -
874 + asdm_import11]
875 +
876 +if __name__ == '__main__':
877 + unittest.main()

Everything looks good. We'll let you know here if there's anything you should know about.

Add shortcut