casa5
Public
Commits
Takeshi Nakazato authored 882a9de0e7b Merge
| 44 44 | default='' means all times. examples: |
| 45 45 | timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' |
| 46 46 | timerange='< YYYY/MM/DD/HH:MM:SS.sss' |
| 47 47 | timerange='> YYYY/MM/DD/HH:MM:SS.sss' |
| 48 48 | timerange='< ddd/HH:MM:SS.sss' |
| 49 49 | timerange='> ddd/HH:MM:SS.sss' |
| 50 50 | scan -- Scan numbers to select. |
| 51 51 | default '' (all). |
| 52 52 | intent -- Scan intents to select. |
| 53 53 | default '' (all). |
| 54 - | array -- (Sub)array IDs to select. |
| 54 + | array -- (Sub)array IDs to select. |
| 55 55 | default '' (all). |
| 56 56 | uvrange -- uv distance range to select. |
| 57 57 | default '' (all). |
| 58 58 | correlation -- Select correlations, e.g. 'rr, ll' or ['XY', 'YX']. |
| 59 59 | default '' (all). |
| 60 60 | observation -- Select by observation ID(s). |
| 61 61 | default '' (all). |
| 62 62 | combine -- Data descriptors that time averaging can ignore: |
| 63 63 | scan, and/or state |
| 64 64 | Default '' (none) |
| 65 65 | keepflags -- Keep flagged data, if possible |
| 66 66 | Default True |
| 67 67 | |
| 68 68 | keepmms -- If the input is a multi-MS, make the output one, too. (experimental) |
| 69 69 | Default: False |
| 70 - | |
| 70 + | |
| 71 71 | """ |
| 72 72 | |
| 73 73 | casalog.origin('oldsplit') |
| 74 + | casalog.post("Warning: the oldsplit task is only for testing. Please use split or mstransform instead.", "WARN") |
| 74 75 | mylocals = locals() |
| 75 76 | rval = True |
| 76 77 | try: |
| 77 78 | |
| 78 - | if (keepmms and ParallelTaskHelper.isParallelMS(vis)): |
| 79 - | if (timebin!='0s' and timebin!='-1s'): |
| 79 + | if (keepmms and ParallelTaskHelper.isParallelMS(vis)): |
| 80 + | if (timebin!='0s' and timebin!='-1s'): |
| 80 81 | casalog.post('Averaging over time with keepmms=True may lead to results different\n' |
| 81 82 | +' from those obtained with keepmms=False due to different binning.', 'WARN') |
| 82 - | |
| 83 + | |
| 83 84 | myms = mstool() |
| 84 85 | myms.open(vis) |
| 85 86 | mses = myms.getreferencedtables() |
| 86 - | myms.close() |
| 87 + | myms.close() |
| 87 88 | mses.sort() |
| 88 89 | |
| 89 90 | nfail = 0 |
| 90 91 | if os.path.exists(outputvis): |
| 91 92 | raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis |
| 92 93 | tempout = outputvis+str(time.time()) |
| 93 94 | os.mkdir(tempout) |
| 94 95 | successfulmses = [] |
| 95 96 | mastersubms = '' |
| 96 97 | masterptab = '' |
| 97 98 | emptyptab = tempout+'/EMPTY_POINTING' |
| 98 99 | nochangeinpointing = (str(antenna)+str(timerange)=='') |
| 99 100 | |
| 100 - | if nochangeinpointing: |
| 101 + | if nochangeinpointing: |
| 101 102 | # resulting pointing table is the same for all |
| 102 103 | # -> replace by empty table if it is a link and won't be modified anyway |
| 103 104 | # and put back original into the master after split |
| 104 105 | |
| 105 106 | # find the master |
| 106 107 | for m in mses: |
| 107 108 | theptab = m+'/POINTING' |
| 108 109 | if not os.path.islink(theptab): |
| 109 110 | #print "is master ", theptab |
| 110 111 | mastersubms = m |
| 118 119 | tmpp.close() |
| 119 120 | del tmpp |
| 120 121 | break |
| 121 122 | |
| 122 123 | mytb = tbtool() |
| 123 124 | |
| 124 125 | # prepare the input MMS for processing |
| 125 126 | replaced = [] |
| 126 127 | outputviss = [] |
| 127 128 | theptabs = [] |
| 128 - | |
| 129 + | |
| 129 130 | for m in mses: |
| 130 131 | |
| 131 132 | # make sure the SORTED_TABLE keywords are disabled |
| 132 133 | mytb.open(m, nomodify=False) |
| 133 134 | if 'SORTED_TABLE' in mytb.keywordnames(): |
| 134 135 | tobedel = mytb.getkeyword('SORTED_TABLE').split(' ')[1] |
| 135 136 | mytb.removekeyword('SORTED_TABLE') |
| 136 137 | os.system('rm -rf '+tobedel) |
| 137 138 | mytb.close() |
| 138 139 | |
| 179 180 | if len(successfulmses)==0: |
| 180 181 | casalog.post('Split failed in all subMSs.', 'WARN') |
| 181 182 | rval=False |
| 182 183 | else: |
| 183 184 | casalog.post('*** Summary: there were failures in '+str(nfail)+' SUBMSs:', 'WARN') |
| 184 185 | casalog.post('*** (these are harmless if they are caused by selection):', 'WARN') |
| 185 186 | for m in mses: |
| 186 187 | if not goretval[m]: |
| 187 188 | casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'WARN') |
| 188 189 | else: |
| 189 - | casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'NORMAL') |
| 190 + | casalog.post(os.path.basename(m)+': '+str(goretval[m]), 'NORMAL') |
| 190 191 | |
| 191 192 | casalog.post('Will construct MMS from subMSs with successful selection ...', 'NORMAL') |
| 192 193 | |
| 193 194 | if nochangeinpointing: # need to take care of POINTING table |
| 194 195 | # in case the master subms did not make it |
| 195 196 | if not (tempout+'/'+os.path.basename(mastersubms) in successfulmses): |
| 196 197 | # old master subms was not selected. |
| 197 198 | # copy the original masterptab into the new master |
| 198 199 | shutil.rmtree(successfulmses[0]+'/POINTING') |
| 199 200 | shutil.copytree(masterptab, successfulmses[0]+'/POINTING') |
| 200 - | |
| 201 + | |
| 201 202 | if rval: # construct new MMS from the output |
| 202 203 | if(width==1 and str(field)+str(spw)+str(antenna)+str(timerange)+str(scan)+str(intent)\ |
| 203 204 | +str(array)+str(uvrange)+str(correlation)+str(observation)==''): |
| 204 205 | ph.makeMMS(outputvis, successfulmses) |
| 205 206 | else: |
| 206 207 | myms.open(successfulmses[0], nomodify=False) |
| 207 208 | auxfile = "split_aux_"+str(time.time()) |
| 208 209 | for i in xrange(1,len(successfulmses)): |
| 209 210 | myms.virtconcatenate(successfulmses[i], auxfile, '1Hz', '10mas', True) |
| 210 211 | myms.close() |
| 211 212 | os.remove(auxfile) |
| 212 - | ph.makeMMS(outputvis, successfulmses, True, ['POINTING']) |
| 213 + | ph.makeMMS(outputvis, successfulmses, True, ['POINTING']) |
| 213 214 | |
| 214 215 | |
| 215 216 | shutil.rmtree(tempout, ignore_errors=True) |
| 216 217 | |
| 217 218 | |
| 218 219 | |
| 219 220 | else: # do not output an MMS |
| 220 221 | |
| 221 222 | rval = split_core(vis, outputvis, datacolumn, field, spw, width, antenna, |
| 222 223 | timebin, timerange, scan, intent, array, uvrange, |
| 223 224 | correlation, observation, combine, keepflags) |
| 224 225 | |
| 225 226 | except Exception, instance: |
| 226 227 | casalog.post("*** Error: %s" % (instance), 'SEVERE') |
| 227 228 | rval = False |
| 228 - | |
| 229 + | |
| 229 230 | |
| 230 231 | return rval |
| 231 232 | |
| 232 233 | def split_core(vis, outputvis, datacolumn, field, spw, width, antenna, |
| 233 234 | timebin, timerange, scan, intent, array, uvrange, |
| 234 235 | correlation, observation, combine, keepflags): |
| 235 236 | |
| 236 237 | retval = True |
| 237 238 | |
| 238 239 | if not outputvis or outputvis.isspace(): |
| 252 253 | if (os.path.exists(outputvis+".flagversions")): |
| 253 254 | myms.close() |
| 254 255 | raise ValueError, "The flagversions \"%s.flagversions\" for the output MS already exist. Please delete." % outputvis |
| 255 256 | |
| 256 257 | # No longer needed. When did it get put in? Note that the default |
| 257 258 | # spw='*' in myms.split ends up as '' since the default type for a variant |
| 258 259 | # is BOOLVEC. (Of course!) Therefore both split and myms.split must |
| 259 260 | # work properly when spw=''. |
| 260 261 | #if(spw == ''): |
| 261 262 | # spw = '*' |
| 262 - | |
| 263 + | |
| 263 264 | if(type(antenna) == list): |
| 264 265 | antenna = ', '.join([str(ant) for ant in antenna]) |
| 265 266 | |
| 266 267 | ## Accept digits without units ...assume seconds |
| 267 268 | timebin = qa.convert(qa.quantity(timebin), 's')['value'] |
| 268 269 | timebin = str(timebin) + 's' |
| 269 - | |
| 270 + | |
| 270 271 | if timebin == '0s': |
| 271 272 | timebin = '-1s' |
| 272 273 | |
| 273 274 | # MSStateGram is picky ('CALIBRATE_WVR.REFERENCE, OBSERVE_TARGET_ON_SOURCE' |
| 274 275 | # doesn't work, but 'CALIBRATE_WVR.REFERENCE,OBSERVE_TARGET_ON_SOURCE' |
| 275 276 | # does), and I don't want to mess with bison now. A .upper() might be a |
| 276 277 | # good idea too, but the MS def'n v.2 does not say whether OBS_MODE should |
| 277 278 | # be case-insensitive. |
| 278 279 | intent = intent.replace(', ', ',') |
| 279 280 | |
| 280 281 | if '^' in spw: |
| 281 282 | casalog.post("The interpretation of ^n in split's spw strings has changed from 'average n' to 'skip n' channels!", 'WARN') |
| 282 283 | casalog.post("Watch for Slicer errors", 'WARN') |
| 283 - | |
| 284 + | |
| 284 285 | if type(width) == str: |
| 285 286 | try: |
| 286 287 | if(width.isdigit()): |
| 287 288 | width=[string.atoi(width)] |
| 288 289 | elif(width.count('[') == 1 and width.count(']') == 1): |
| 289 290 | width = width.replace('[', '') |
| 290 291 | width = width.replace(']', '') |
| 291 292 | splitwidth = width.split(',') |
| 292 293 | width = [] |
| 293 294 | for ws in splitwidth: |
| 294 295 | if(ws.isdigit()): |
| 295 - | width.append(string.atoi(ws)) |
| 296 + | width.append(string.atoi(ws)) |
| 296 297 | else: |
| 297 298 | width = [1] |
| 298 299 | except: |
| 299 300 | raise TypeError, 'parameter width is invalid...using 1' |
| 300 301 | |
| 301 302 | if type(correlation) == list: |
| 302 303 | correlation = ', '.join(correlation) |
| 303 304 | correlation = correlation.upper() |
| 304 305 | |
| 305 306 | if hasattr(combine, '__iter__'): |
| 342 343 | whichcol=datacolumn, |
| 343 344 | scan=scan, uvrange=uvrange, |
| 344 345 | combine=combine, |
| 345 346 | correlation=correlation, intent=intent, |
| 346 347 | obs=str(observation)): |
| 347 348 | myms.close() |
| 348 349 | if os.path.isdir(cavms): |
| 349 350 | import shutil |
| 350 351 | shutil.rmtree(cavms) |
| 351 352 | return False |
| 352 - | |
| 353 + | |
| 353 354 | # The selection was already made, so blank them before time averaging. |
| 354 355 | field = '' |
| 355 356 | spw = '' |
| 356 357 | width = [1] |
| 357 358 | antenna = '' |
| 358 359 | array = '' |
| 359 360 | timerange = '' |
| 360 361 | datacolumn = 'all' |
| 361 362 | scan = '' |
| 362 363 | intent = '' |
| 386 387 | return False |
| 387 388 | myms.close() |
| 388 389 | |
| 389 390 | if do_both_chan_and_time_mod: |
| 390 391 | import shutil |
| 391 392 | shutil.rmtree(cavms) |
| 392 393 | |
| 393 394 | # Write history to output MS, not the input ms. |
| 394 395 | try: |
| 395 396 | param_names = split_core.func_code.co_varnames[:split_core.func_code.co_argcount] |
| 396 - | param_vals = [eval(p) for p in param_names] |
| 397 + | param_vals = [eval(p) for p in param_names] |
| 397 398 | retval &= write_history(myms, outputvis, 'oldsplit', param_names, param_vals, |
| 398 399 | casalog) |
| 399 400 | except Exception, instance: |
| 400 401 | casalog.post("*** Error \'%s\' updating HISTORY" % (instance), |
| 401 402 | 'WARN') |
| 402 403 | |
| 403 404 | # Update FLAG_CMD if necessary. |
| 404 - | # If the spw selection is by name or FLAG_CMD contains spw with names, skip the updating |
| 405 + | # If the spw selection is by name or FLAG_CMD contains spw with names, skip the updating |
| 405 406 | |
| 406 407 | if ((spw != '') and (spw != '*')) or do_chan_mod: |
| 407 408 | isopen = False |
| 408 409 | mytb = tbtool() |
| 409 410 | try: |
| 410 411 | mytb.open(outputvis + '/FLAG_CMD', nomodify=False) |
| 411 412 | isopen = True |
| 412 413 | nflgcmds = mytb.nrows() |
| 413 - | |
| 414 + | |
| 414 415 | if nflgcmds > 0: |
| 415 416 | updateFlagCmd = False |
| 416 417 | # If spw selection is by name in FLAG_CMD, do not update, CAS-7751 |
| 417 418 | mycmd = mytb.getcell('COMMAND', 0) |
| 418 419 | cmdlist = mycmd.split() |
| 419 420 | for cmd in cmdlist: |
| 420 421 | # Match only spw indices, not names |
| 421 422 | if cmd.__contains__('spw'): |
| 422 423 | cmd = cmd.strip("spw=") |
| 423 424 | spwstr = re.search('^[^a-zA-Z]+$', cmd) |
| 424 425 | if spwstr != None and spwstr.string.__len__() > 0: |
| 425 426 | updateFlagCmd = True |
| 426 - | break |
| 427 - | |
| 428 - | |
| 429 - | if updateFlagCmd: |
| 427 + | break |
| 428 + | |
| 429 + | |
| 430 + | if updateFlagCmd: |
| 430 431 | mademod = False |
| 431 432 | cmds = mytb.getcol('COMMAND') |
| 432 433 | widths = {} |
| 433 434 | #print "width =", width |
| 434 435 | if hasattr(width, 'has_key'): |
| 435 436 | widths = width |
| 436 437 | else: |
| 437 438 | if hasattr(width, '__iter__') and len(width) > 1: |
| 438 439 | for i in xrange(len(width)): |
| 439 440 | widths[i] = width[i] |
| 440 441 | elif width != 1: |
| 441 442 | #print 'using myms.msseltoindex + a scalar width' |
| 442 443 | nspw = len(myms.msseltoindex(vis=vis, |
| 443 444 | spw='*')['spw']) |
| 444 445 | if hasattr(width, '__iter__'): |
| 445 446 | w = width[0] |
| 446 447 | else: |
| 447 448 | w = width |
| 448 449 | for i in xrange(nspw): |
| 449 450 | widths[i] = w |
| 450 - | #print 'widths =', widths |
| 451 + | #print 'widths =', widths |
| 451 452 | for rownum in xrange(nflgcmds): |
| 452 453 | # Matches a bare number or a string quoted any way. |
| 453 454 | spwmatch = re.search(r'spw\s*=\s*(\S+)', cmds[rownum]) |
| 454 455 | if spwmatch: |
| 455 456 | sch1 = spwmatch.groups()[0] |
| 456 457 | sch1 = re.sub(r"[\'\"]", '', sch1) # Dequote |
| 457 458 | # Provide a default in case the split selection excludes |
| 458 459 | # cmds[rownum]. update_spwchan() will throw an exception |
| 459 460 | # in that case. |
| 460 461 | cmd = '' |
| 481 482 | mademod = True |
| 482 483 | cmds[rownum] = cmd |
| 483 484 | if mademod: |
| 484 485 | casalog.post('Updating FLAG_CMD', 'INFO') |
| 485 486 | mytb.putcol('COMMAND', cmds) |
| 486 487 | |
| 487 488 | else: |
| 488 489 | casalog.post('FLAG_CMD table contains spw selection by name. Will not update it!','DEBUG') |
| 489 490 | |
| 490 491 | mytb.close() |
| 491 - | |
| 492 + | |
| 492 493 | except Exception, instance: |
| 493 494 | if isopen: |
| 494 495 | mytb.close() |
| 495 496 | myms = None |
| 496 497 | mytb = None |
| 497 498 | casalog.post("*** Error \'%s\' updating FLAG_CMD" % (instance), |
| 498 499 | 'SEVERE') |
| 499 500 | return False |
| 500 501 | |
| 501 502 | myms = None |