import os import time import ast import copy import numpy import inspect from collections import deque,defaultdict from collections import OrderedDict from casatasks import casalog, flagdata from casatools import table,quanta,ms,agentflagger from .parallel.parallel_task_helper import ParallelTaskHelper ###some helper tools tblocal = table() mslocal = ms() qalocal = quanta() ''' A set of helper functions for the tasks flagdata and flagcmd. Class Parser: to parse flag commands I/O functions: get_flag_cmd_list readFile readFiles readAndParse parseDictionary parseXML readAntennaList writeAntennaList writeFlagCommands writeRflagThresholdFile Parameter handling compressSelectionList evalParams selectReason parseSelectionPars parseUnion purgeEmptyPars purgeParameter parseAgents Others backupFlags convertDictToString convertStringToDict extractAntennaInfo save_rflag_consolidated_files parseRflagOutputFromSummary ''' debug = False def get_task_arg_default( func, arg ): print(arg) spec = inspect.getfullargspec(func.__call__) if arg not in spec.args: raise Exception("cannot find '%s' among the function arguments" % arg) return spec.defaults[spec.args.index(arg)-1] class Parser(): ''' Parser for input files. primarydivider --> first split the string by this character secondarydivider --> next split the string by this character The constructor takes two separators. It first splits the string by the 'primarydivider'. It then loops through each entry after the first split and verifies if the 'secondarydivider' is in the string. If yes, the string is added to a list. If not, it removes the whitespace of the initial split, by bringing back two strings together. This will allow the primarydivider to be part of the string itself. Last thing, it returns an ordered dictionary using the imported class OrderedDict. ''' def __init__(self,primarydivider,secondarydivider): self.prime = primarydivider self.second = secondarydivider def parse2Dictionary(self,string): res = self.initialsplit(string) new = [] for entry in res: if self.second not in entry: new[-1] += ' ' + entry else: new.append(entry) # return dict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new]) return OrderedDict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new]) def parseNoEval(self,string): res = self.initialsplit(string) new = [] for entry in res: if self.second not in entry: new[-1] += ' ' + entry else: new.append(entry) # return dict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new]) return OrderedDict((entry[0],entry[1]) for entry in [entry.split(self.second,1) for entry in new]) def parse2List(self,string): res = self.initialsplit(string) new = [] for entry in res: if self.second not in entry: new[-1] += ' ' + entry else: new.append(entry) return new # return dict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new]) # return OrderedDict((entry[0],ast.literal_eval(entry[1])) for entry in [entry.split(self.second,1) for entry in new]) def initialsplit(self,string): nstring = string.strip() return nstring.split(self.prime) ####################################################### # # Reading functions # ####################################################### def isCalTable(msname): '''Check if a file is a cal table msname --> filename Return 1 for cal table, 0 for MS and 2 for MMS''' try: tblocal.open(msname) except: raise ValueError("Unable to open MS %s" % msname) tbinfo = tblocal.info() tblocal.close() retval = 0 if tbinfo['type'] == 'Calibration': retval = 1 elif tbinfo['type'] == 'Measurement Set': # MMS type if tbinfo['subType'] == 'CONCATENATED' and ParallelTaskHelper.isParallelMS(msname): retval = 2 else: # MS type retval = 0 else: retval = 0 return retval def addAbsolutePath(input_file): '''Read in the lines from an input file input_file --> file in disk with a list of strinZeeuwgs per line Return a new file saved to disk with relative file names changed to absolute file names. The new file will have the name of the input_file + '.tmp' ''' # Get a list of the flag commands from the file cmdlist = readFile(input_file) # Make a dictionary of the flag commands flagdict = parseDictionary(cmdlist, 'any', False) for key in flagdict: cmddict = flagdict[key]['command'] if 'addantenna' in cmddict and isinstance(cmddict['addantenna'],str) and \ cmddict['addantenna'] != '': cmddict['addantenna'] = os.path.abspath(cmddict['addantenna']) if 'timedev' in cmddict and isinstance(cmddict['timedev'],str) and \ cmddict['timedev'] != '': cmddict['timedev'] = os.path.abspath(cmddict['timedev']) if 'freqdev' in cmddict and isinstance(cmddict['freqdev'],str) and \ cmddict['freqdev'] != '': cmddict['freqdev'] = os.path.abspath(cmddict['freqdev']) # Convert the dictionary back to a list and save it output_file = os.path.abspath(input_file + ".tmp") writeFlagCommands('', flagdict, False, '', output_file, append=False) # Return the temporary file return output_file def get_flag_cmd_list(inpfile): """ For flagdata list mode, get the list of commands from whatever has been given in the inpfile input parameter (a file, a list of files, or a Python string with commands). The output list has one item (command) per line in the input file(s) or string. :param inpfile: inpfile parameter as passed to task flagdata :returns: list of commands found in file(s) or string """ # inpfile is a file if isinstance(inpfile, str) and os.path.isfile(inpfile): flaglist = readFile(inpfile) nlines = len(flaglist) casalog.post('Read %s command(s) from file: %s'%(nlines, inpfile)) # inpfile is a list of files elif isinstance(inpfile, list) and os.path.isfile(inpfile[0]): flaglist = readFiles(inpfile) # Python list of strings elif isinstance(inpfile, list): flaglist = inpfile else: raise ValueError('Unsupported input list of flag commands or input file does not ' 'exist') return flaglist def readFile(inputfile): '''Read in the lines from an input file inputfile --> file in disk with a list of strings per line Returns a list. Blank lines are skipped. Boolean values will be capitalized to avoid errors in the parser later. ''' flagfile = inputfile if (type(flagfile) == str) & os.path.exists(flagfile): try: ff = open(flagfile, 'r') except: casalog.post('Error opening file ' + flagfile,'ERROR') raise else: casalog.post('ASCII file not found - please verify the name','ERROR') raise Exception('ASCII file not found') # Parse file try: cmdlist = [] for line in ff: cmd = line.rstrip() if cmd == '': continue if cmd.startswith('#'): continue uppercmd = cmd.replace('true','True') cmd = uppercmd.replace('false','False') cmdlist.append(cmd) except: casalog.post('Error reading lines from file '+ff.name, 'SEVERE') ff.close() raise ff.close() return cmdlist def readFiles(inputfiles): '''Read in a list of files with flag commands inputfiles --> list of files in disk Returns all files concatenated into one single list. Blank lines are skipped. Boolean values will be capitalized to avoid errors in the parser later. ''' if not isinstance(inputfiles, list): casalog.post('Error opening list of flag commands ' + inputfiles,'ERROR') raise cmdlist = [] # Read files for flagfile in inputfiles: cmd = readFile(flagfile) nlines = len(cmd) casalog.post('Read %s command(s) from file: %s'%(nlines, flagfile)) cmdlist = cmdlist + cmd return cmdlist def readAndParse(inputlist, tbuff=None): '''Read in a list of flag commands and parse them into a dictionary. The flag commands can be from a list of files or from a list of flag commands. If tbuff is given, it will be applied to the timerange parameters. Note: when tbuff is None, it is best to use the individual functions readFile(s) and parseDictionary to avoid parsing the file twice! inputlist --> list of files in disk containing the flag commands or a list of Python strings with flag commands. tbuff --> list of time buffers to apply to each timerange. If tbuff is a list, it should be the same size of the list of inputfiles. If it is a Double value, it will be applied only to the first input file. Returns a list of dictionaries. Blank lines are skipped. Boolean values will be capitalized to avoid errors in the parser. Each parameter=value will become a key=value in the dictionary. Example: inputlist = ["mode='manual' spw='0' autocorr=true", "mode='shadow'"] Returns: [{'mode':'manual','spw':'0','autocorr':True}, {'mode':'shadow'}] ''' if not isinstance(inputlist, list): casalog.post('Error opening list of flag commands ' + inputlist,'ERROR') raise # List of files if os.path.isfile(inputlist[0]): isFile = True # List of strings else: isFile = False if tbuff is None: doPadding = False else: doPadding = True # Make the list of tbuff a deque # dtbuff = deque() # # if isinstance(tbuff, float): # dtbuff.append(tbuff) # elif isinstance(tbuff, list): # dtbuff = deque(i for i in tbuff) # List of dictionaries to return listofdict = [] # Initialize the parser myParser = Parser(' ', '=') # Read files if isFile: for flagfile in inputlist: cmdlist = readFile(flagfile) nlines = len(cmdlist) casalog.post('Read %s command(s) from file: %s'%(nlines, flagfile)) parsedlist = [] for cmd in cmdlist: #Get a dictionary without type evaluation preparsing = myParser.parseNoEval(cmd) # Evaluate the types parsed = evaluateParameters(preparsing) parsedlist.append(parsed) # Apply time buffer to file if doPadding: # mytbuff = dtbuff.popleft() applyTimeBufferList(parsedlist, tbuff) # if dtbuff.__len__() == 0: # doPadding = False listofdict = listofdict + parsedlist # It is a list of strings else: cmdlist = inputlist nlines = len(cmdlist) casalog.post('Read %s command(s) from a Python list of strings'%nlines) parsedlist = [] for cmd in cmdlist: #Get a dictionary without type evaluation preparsing = myParser.parseNoEval(cmd) # Evaluate the types parsed = evaluateParameters(preparsing) parsedlist.append(parsed) # Apply time buffer to list if doPadding: # mytbuff = dtbuff.popleft() applyTimeBufferList(parsedlist, tbuff) listofdict = listofdict + parsedlist return listofdict # Use readAndParse to apply tbuff, which is slightly faster. For # small files, the difference is not noticeable. def readParseTbuff(inputlist, tbuff=None): '''Read in a list of flag commands and parse them into a dictionary. The flag commands can be from a list of files or from a list of flag commands. If tbuff is given, it will be applied to the timerange parameters. Note: when tbuff is None, it is best to use the individual functions readFile(s) and parseDictionary to avoid parsing the file twice! inputlist --> list of files in disk containing the flag commands or a list of Python strings with flag commands. tbuff --> list of time buffers to apply to each timerange. If tbuff is a list, it should be the same size of the list of inputfiles. If it is a Double value, it will be applied only to the first input file. Returns a list of dictionaries. Blank lines are skipped. Boolean values will be capitalized to avoid errors in the parser. Each parameter=value will become a key=value in the dictionary. Example: inputlist = ["mode='manual' spw='0' autocorr=true", "mode='shadow'"] Returns: [{'mode':'manual','spw':'0','autocorr':True}, {'mode':'shadow'}] ''' if not isinstance(inputlist, list): casalog.post('Error opening list of flag commands ' + inputlist,'ERROR') raise # List of files isFile = bool(os.path.isfile(inputlist[0])) if tbuff is None: doPadding = False else: doPadding = True # Make the list of tbuff a deque dtbuff = deque() if isinstance(tbuff, float): dtbuff.append(tbuff) elif isinstance(tbuff, list): dtbuff = deque(i for i in tbuff) # List of dictionaries to return listofdict = [] # Initialize the parser myParser = Parser(' ', '=') # Read files if isFile: for flagfile in inputlist: cmdlist = readFile(flagfile) nlines = len(cmdlist) casalog.post('Read %s command(s) from file: %s'%(nlines, flagfile)) if doPadding: mytbuff = dtbuff.popleft() parsedlist = [] for cmd in cmdlist: #Get a dictionary without type evaluation preparsing = myParser.parseNoEval(cmd) # Evaluate the types parsed = evaluateParameters(preparsing) # Apply time buffer to single command if doPadding: applyTimeBuffer(parsed, mytbuff) parsedlist.append(parsed) if dtbuff.__len__() == 0: doPadding = False # Append dictionary to list listofdict += parsedlist # It is a list of strings else: cmdlist = inputlist nlines = len(cmdlist) casalog.post('Read %s command(s) from a Python list of strings'%nlines) if doPadding: mytbuff = dtbuff.popleft() parsedlist = [] for cmd in cmdlist: #Get a dictionary without type evaluation preparsing = myParser.parseNoEval(cmd) # Evaluate the types parsed = evaluateParameters(preparsing) # Apply time buffer to single command if doPadding: applyTimeBuffer(parsed, mytbuff) parsedlist.append(parsed) listofdict += parsedlist return listofdict def applyTimeBuffer(cmddict, tbuff): ''' Apply in-place a time buffer to timerange from a dictionary with one flag command cmddict --> dictionary with a single flag command: Ex: {'antenna':'DV01', 'timerange':'2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'} tbuff --> float value of time buffer to apply to all timerange parameters * it assumes that timerange has syntax t0~t1 * split timerange in '~' to get t0 and t1 * convert value to time in days using qalocal.totime * convert days to seconds * subtract tbuff from t0 and add tbuff to t1 * convert back to time string with the form 'ymd' using qalocal.time * write new values back to input dictionary ''' # if not isinstance(tbuff, float): # casalog.post('Time buffer (tbuff) is not of type float', 'WARN') # return if 'timerange' in cmddict: timerange = cmddict['timerange'] if timerange.find('~') != -1: t0,t1 = timerange.split('~',1) # start time startTime = qalocal.totime(t0)['value'] startTimeSec = (startTime * 24 * 3600) - tbuff startTimeSec = qalocal.quantity(startTimeSec, 's') paddedT0 = qalocal.time(startTimeSec,form='ymd',prec=9)[0] # end time endTime = qalocal.totime(t1)['value'] endTimeSec = (endTime * 24 * 3600) + tbuff endTimeSec = qalocal.quantity(endTimeSec, 's') paddedT1 = qalocal.time(endTimeSec,form='ymd',prec=9)[0] # update the original dictionary cmddict['timerange'] = paddedT0+'~'+paddedT1 return def applyTimeBufferList(alist, tbuff=None): ''' Apply in-place a time buffer to ALL timerange parameters of a list of dictionaries with several flag commands. It will do the following: alist --> list of dictionaries with flag commands. Ex: [{'antenna':'DV01', 'timerange':'2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'}, {'antenna':'DV02', 'timerange':'2013/10/15/10:25:30.110~2013/10/15/10:25:32.230'}, ...] tbuff --> float value or list of 2 values of time buffer to apply to all timerange parameters. When tbuff is a list of 2 values, the first value is applied to the lower time, the second to the upper time. * it assumes that timerange has syntax t0~t1 * split timerange in '~' to get t0 and t1 * convert value to time in days using qalocal.totime * convert days to seconds * subtract tbuff0 from t0 and add tbuff1 to t1 * convert back to time string with the form 'ymd' using qalocal.time * write new values back to dictionary ''' # if not isinstance(tbuff, float): # casalog.post('Time buffer (tbuff) is not of type float', 'WARN') # return casalog.post('Apply time buffer padding to list of dictionaries', 'DEBUG1') # When tbuff is float, the range is regular, otherwise it's irregular if isinstance(tbuff, list) and len(tbuff) == 2: tbuff0 = tbuff[0] tbuff1 = tbuff[1] elif isinstance(tbuff, list) and len(tbuff) == 1: tbuff0 = tbuff1 = tbuff[0] elif isinstance(tbuff, float): tbuff0 = tbuff1 = tbuff else: casalog.post('Time buffer (tbuff) is not of type float or list', 'WARN') return for cmddict in alist: if 'timerange' in cmddict: timerange = cmddict['timerange'] if timerange.find('~') != -1: t0,t1 = timerange.split('~',1) # start time startTime = qalocal.totime(t0)['value'] startTimeSec = (startTime * 24 * 3600) - tbuff0 startTimeSec = qalocal.quantity(startTimeSec, 's') paddedT0 = qalocal.time(startTimeSec,form='ymd',prec=9)[0] # end time endTime = qalocal.totime(t1)['value'] endTimeSec = (endTime * 24 * 3600) + tbuff1 endTimeSec = qalocal.quantity(endTimeSec, 's') paddedT1 = qalocal.time(endTimeSec,form='ymd',prec=9)[0] # update the original dictionary cmddict['timerange'] = paddedT0+'~'+paddedT1 return def parseDictionary(cmdlist, reason='any', shadow=True): '''Create a dictionary after parsing a list of flag commands. If reason is different than 'any', only the selected rows will be parsed to the final dictionary. cmdlist --> list of flag commands OR list of dictionaries with flag commands as key:value reason --> reason or list of reasons to select from shadow --> True will make a dictionary of the addantenna parameter and add it to the flag command Returns a dictionary with the the selected rows. Each parameter will become a key in the dictionary. If a mode='shadow' is present in the dictionary and the parameter addantenna is a filename (string), it will create a dictionary of the filename and add it to the parsed dictionary. ''' if cmdlist.__len__() == 0: raise Exception('Empty list of commands') # Gather all requested reasons myreaslist = [] if type(reason) == str: if reason != 'any': myreaslist.append(reason) elif type(reason) == list: myreaslist = reason else: casalog.post('Cannot read reason; it contains unknown variable types', 'ERROR') return # Separate per ' ', then per '=' myParser = Parser(' ', '=') flagdict = {} row = 0 for cmd in cmdlist: cmddict = {} # Simple list of strings ([key1='value1' key2='value2'] if isinstance(cmd, str): # Skip comment and empty lines if cmd.startswith('#'): continue if cmd == '': continue uppercmd = cmd.replace('true','True') cmd = uppercmd.replace('false','False') # Get a dictionary without type evaluation preparsing = myParser.parseNoEval(cmd) # Evaluate the types parsed = evaluateParameters(preparsing) # List of dictionaries: [{'key1':'value1', 'key2':'value2'}] elif isinstance(cmd, dict): parsed = cmd # Parse the flag commands into a dictionary mode = '' antenna = '' timerange = '' cmddict['row'] = str(row) cmddict['id'] = str(row) cmddict['command'] = parsed if 'mode' in parsed: mode = parsed['mode'] input_reason = '' if 'reason' in parsed: input_reason = parsed['reason'] cmddict['reason'] = input_reason if 'timerange' in parsed: timerange = parsed['timerange'] if 'antenna' in parsed: antenna = parsed['antenna'] cmddict['applied'] = False cmddict['time'] = 0.0 cmddict['interval'] = 0 cmddict['level'] = 0 cmddict['severity'] = 0 cmddict['type'] = '' cmddict['mode'] = mode cmddict['timerange'] = timerange cmddict['antenna'] = antenna flagdict[row] = cmddict row += 1 # Select the input cmds based on the requested reasons selected_dict = {} cmddict = {} row = 0 if myreaslist.__len__() > 0: for key in flagdict: cmddict = flagdict[key]['command'] if 'reason' in cmddict: input_reason = cmddict['reason'] if selectReason(myreaslist, input_reason): selected_dict[row] = flagdict[key] row += 1 if selected_dict.__len__() == 0: raise Exception('No input lines matching requested reason(s)') else: selected_dict = flagdict # Specific for the shadow mode if shadow: cmddict = {} for key in selected_dict: cmddict = selected_dict[key]['command'] if ( 'addantenna' in cmddict and isinstance(cmddict['addantenna'], str) and cmddict['addantenna'] != '' ): # Create a dictionary and replace the parameter casalog.post('The addantenna parameter will be parsed as a dictionary', 'DEBUG1') antdict = readAntennaList(cmddict['addantenna']) selected_dict[key]['command']['addantenna'] = antdict return selected_dict def selectReason(reasonlist, pattern): '''Return True if pattern is in the reasonlist''' if isinstance(reasonlist, str): if pattern == reasonlist: return True elif isinstance(reasonlist, list): for r in reasonlist: if pattern == r: return True return False def parseSelectionPars(cmddict): '''Return only the selection parameters into a dictionary: cmddict --> one row-dictionary with parameters The correlation parameter will not be considered. ''' # Only these parameters will be included in dictionary # correlation is handled by the agents parlist = ['observation','array','feed','scan','field','spw', 'timerange','uvrange','intent','antenna'] return {par: cmddict[par] for par in parlist if par in cmddict} def parseUnion(vis, flagdict): '''Get a dictionary of a union of all selection parameters from a dictionary of flag commands. vis --> MS flagdict --> dictionary of parameters and values (par=val) such as the one returned by parseDictionary() Returns a new dictionary with the union of the selection parameters. ''' # Union dictionary to return dictpars = {} # Strings for each parameter scans = '' fields = '' ants = '' times = '' corrs = '' ints = '' feeds = '' arrays = '' uvs = '' spws = '' obs = '' # Counters for each parameter si = 0 # count the number of lines with scan fi = 0 # count the number of lines with field ai = 0 # count the number of lines with antenna ti = 0 # count the number of lines with timerange coi = 0 # count the number of lines with correlation ii = 0 # count the number of lines with intent fei = 0 # count the number of lines with feed ari = 0 # count the number of lines with array ui = 0 # count the number of lines with uvrange pi = 0 # count the number of lines with spw oi = 0 # count the number of lines with observation for k in flagdict.keys(): # Each key is a dictionary of one flag command cmddict = flagdict[k]['command'] for xkey,xval in cmddict.items(): # Check which parameter if xkey == "scan": scans += xval + ',' si += 1 elif xkey == "field": fields += xval + ',' fi += 1 elif xkey == "antenna": ants += xval + ';' ai += 1 elif xkey == "timerange": times += xval + ',' ti += 1 elif xkey == "correlation": corrs += xval + ',' coi += 1 elif xkey == "intent": ints += xval + ',' ii += 1 elif xkey == "feed": feeds += xval + ',' fei += 1 elif xkey == "array": arrays += xval + ',' ari += 1 elif xkey == "uvrange": uvs += xval + ',' ui += 1 elif xkey == "spw": spws += xval + ',' pi += 1 elif xkey == "observation": obs += xval + ',' oi += 1 # Dictionary with number of occurrences of each parameter npars = { 'field':fi, 'scan':si, 'antenna':ai, 'spw':pi, 'timerange':ti, 'correlation':coi, 'intent':ii, 'feed':fei, 'array':ari, 'uvrange':ui, 'observation':oi } # Strip out the extra comma at the end scans = scans.rstrip(',') fields = fields.rstrip(',') ants = ants.rstrip(';') times = times.rstrip(',') corrs = corrs.rstrip(',') ints = ints.rstrip(',') feeds = feeds.rstrip(',') arrays = arrays.rstrip(',') uvs = uvs.rstrip(',') spws = spws.rstrip(',') obs = obs.rstrip(',') dictpars['scan'] = scans dictpars['field'] = fields # Antennas are handled better within the framework. dictpars['antenna'] = '' # Times are handled better within the framework. dictpars['timerange'] = '' # Correlations should be handled only by the agents dictpars['correlation'] = '' # CAS-4682, do not create union for intents dictpars['intent'] = '' dictpars['feed'] = feeds dictpars['array'] = arrays dictpars['uvrange'] = uvs dictpars['spw'] = spws dictpars['observation'] = obs # Compress the selection list to reduce MSSelection parsing time. # 'field','spw','antenna' strings in dicpars will be modified in-place. compressSelectionList(vis,dictpars); # The number of keys in flagdict is the number of flag commands ncmds = flagdict.__len__() # Make the union. Only leave non-empty parameters in dictionary for k,v in npars.items(): if v < ncmds: dictpars.pop(k) return dictpars def _merge_timerange(commands): ''' merge manual commands that only differ in timerange and agentname this speeds up manual flagging using large lists cmd -> list of flagging commands returns list of commands with unique key ''' merged = dict() lunique = [] for cmd in commands: try: # only merge manual commands with timerange if (cmd.get('mode') != 'manual') or ('timerange' not in cmd): raise ValueError # create sorted list of command keys excluding agentname which # changes for each manual flag compound_key = sorted(x for x in cmd.keys() if x not in ('timerange', 'agentname')) # create compound key of all command keys and their values (e.g. antenna:1) compound = tuple((x, cmd[x]) for x in compound_key) # skip invalid timeranges so they don't remove the whole agent group if '~' in cmd['timerange']: t0,t1 = cmd['timerange'].split('~', 1) startTime = qalocal.totime(t0)['value'] endTime = qalocal.totime(t1)['value'] if endTime <= startTime: raise ValueError # merge timerange duplicate compound keys try: merged[compound]['timerange'] += ',' + cmd['timerange'] except KeyError: merged[compound] = copy.deepcopy(cmd) except: # add merged keys to non-mergeable keys, also on errors like # non-hashable keys lunique.extend(merged.values()) # append non mergeable key lunique.append(copy.deepcopy(cmd)) # reset merge to preserve ordering of manual and other flags # e.g. summary,manual,manual,manual,summary,manual # to summary,merged-manual,summary,manual merged = dict() # add remaining merged keys to non-mergeable keys lunique.extend(merged.values()) return lunique def parseAgents(aflocal, flagdict, myrows, apply, writeflags, display=''): ''' Setup the parameters of each agent and call the agentflagger tool aflocal --> local instance of the agentflagger tool (assumes the MS is open) flagdict --> a dictionary of the flag commands to parse to tool myrows --> selected rows to apply/unapply flags apply --> a boolean to control whether to apply or unapply the flags writeflags --> used by mode=rflag only display --> used by mode='rflag' only The original flagdict dictionary is not modified ''' if flagdict.__len__() <= 0: casalog.post('There are no flag cmds in list', 'SEVERE') return False # Do not modify original dictionary myflagcmd = copy.deepcopy(flagdict) commands = [] # Setup the agent for each input line and merge timeranges to one command for row in myflagcmd.keys(): cmd = OrderedDict() cmd = myflagcmd[row]['command'] if 'reason' in cmd: cmd.pop('reason') if ( 'mode' not in cmd or cmd['mode'] == '' or cmd['mode'] == 'manualflag' ): cmd['mode'] = 'manual' elif cmd['mode'] == 'rflag': cmd['writeflags'] = writeflags cmd['display'] = display # Read ntime readNtime(cmd) # Cast the correct type of some parameters evalParams(cmd) # Add the apply/unapply parameter to dictionary cmd['apply'] = apply # Hold the name of the agent and the cmd row number mode = cmd['mode'] agent_name = mode.capitalize()+'_'+str(row) cmd['agentname'] = agent_name # Remove the data selection parameters if there is only one agent for performance reasons. # Explanation: if only one agent exists and the data selection parameters are parsed to it, # it will have to go through the entire MS and check if each data selection given to the agent # matches what the user asked in the selected data. # Only correlation, antenna and timerange will go to the agent # CAS-3959 Handle channel selection at the FlagAgent level, leave spw in here too if myflagcmd.__len__() == 1: sellist=['scan','field','intent','feed','array','uvrange','observation'] for k in sellist: if k in cmd: cmd.pop(k) casalog.post('Parsing parameters of mode %s in row %s'%(mode,row), 'DEBUG') casalog.post('%s'%cmd,'DEBUG') commands.append(cmd) merged = _merge_timerange(commands) if len(myflagcmd) != len(merged): casalog.post('Reduced %d timerange flags into %d compound flags' % (len(myflagcmd), len(merged))) for cmd in merged: # Parse the dictionary of parameters to the tool if (not aflocal.parseagentparameters(cmd)): casalog.post('Failed to parse parameters of mode %s in row %s' %(mode,row), 'WARN') continue # return True return myflagcmd # TO BE VERIFIED. May not be needed def evalParams(params): '''Give correct types to non-string parameters The types are defined in the XML file of the task flagdata Do not repeat any parameter''' # manual parameter # if 'autocorr' in params: # params['autocorr'] = eval(params['autocorr'].capitalize()) # quack parameters # if 'quackmode' in params and not params['quackmode'] in ['beg' # , 'endb', 'end', 'tail']: # raise Exception, \ # "Illegal value '%s' of parameter quackmode, must be either 'beg', 'endb', 'end' or 'tail'" \ # % params['quackmode'] # if 'quackinterval' in params: # params['quackinterval'] = float(params['quackinterval']) # if 'quackincrement'in params: # if type(params['quackincrement']) == str: # params['quackincrement'] = eval(params['quackincrement'].capitalize()) # clip parameters if 'clipminmax' in params: val1 = params['clipminmax'][0] val2 = params['clipminmax'][1] params['clipminmax'] = [float(val1), float(val2)] # if 'clipoutside' in params: # if type(params['clipoutside']) == str: # params['clipoutside'] = eval(params['clipoutside'].capitalize()) # else: # params['clipoutside'] = params['clipoutside'] # if 'channelavg' in params: # params['channelavg'] = eval(params['channelavg'].capitalize()) # if 'clipzeros' in params: # params['clipzeros'] = eval(params['clipzeros'].capitalize()) # shadow parameter # if 'tolerance' in params: # params['tolerance'] = float(params['tolerance']) # # # elevation parameters # if 'lowerlimit' in params: # params['lowerlimit'] = float(params['lowerlimit']) # if 'upperlimit' in params: # params['upperlimit'] = float(params['upperlimit']) # extend parameters # if 'extendpols' in params: # params['extendpols'] = eval(params['extendpols'].capitalize()) # if 'growtime' in params: # params['growtime'] = float(params['growtime']) # if 'growfreq' in params: # params['growfreq'] = float(params['growfreq']) # if 'growaround' in params: # params['growaround'] = eval(params['growaround'].capitalize()) # if 'flagneartime' in params: # params['flagneartime'] = eval(params['flagneartime'].capitalize()) # if 'flagnearfreq' in params: # params['flagnearfreq'] = eval(params['flagnearfreq'].capitalize()) # tfcrop parameters # if 'combinescans' in params: # params['combinescans'] = eval(params['combinescans'].capitalize()) # if 'timecutoff' in params: # params['timecutoff'] = float(params['timecutoff']) # if 'freqcutoff' in params: # params['freqcutoff'] = float(params['freqcutoff']) # if 'maxnpieces' in params: # params['maxnpieces'] = int(params['maxnpieces']) # if 'halfwin' in params: # params['halfwin'] = int(params['halfwin']) # if 'extendflags' in params: # params['extendflags'] = eval(params['extendflags'].capitalize()) # rflag parameters # if 'winsize' in params: # params['winsize'] = int(params['winsize']); # This is only necessary when timedev/freqdev are strings if 'timedev' in params: timepar = params['timedev'] if isinstance(timepar, list): return try: timepar = eval(timepar) except Exception: timepar = readRFlagThresholdFile(params['timedev'],'timedev'); params['timedev'] = timepar if 'freqdev' in params: freqpar = params['freqdev'] if isinstance(freqpar, list): return try: freqpar = eval(freqpar) except Exception: freqpar = readRFlagThresholdFile(params['freqdev'],'freqdev'); params['freqdev'] = freqpar # if 'timedevscale' in params: # params['timedevscale'] = float(params['timedevscale']); # if 'freqdevscale' in params: # params['freqdevscale'] = float(params['freqdevscale']); # if 'spectralmin' in params: # params['spectralmin'] = float(params['spectralmin']); # if 'spectralmax' in params: # params['spectralmax'] = float(params['spectralmax']); def writeFlagCommands(msfile, flagdict, applied, add_reason, outfile, append=True): ''' Writes the flag commands to an ASCII file or to the FLAG_CMD table msfile --> MS name to save to FLAG_CMD table or empty. flagdict --> dictionary of commands read from inputfile applied --> True or False to write to APPLIED column add_reason --> reason to add to output (replace input reason, if any) outfile --> if not empty, save to it append --> if True it will append to outfile, otherwise it removes outfile before saving to it. ''' try: import pylab as pl except ImportError as exc: raise ImportError('Failed to load pylab, required in writeFlagCommands: ', exc) reason2add = False # Replace blanks from reason and cmdreason with underscores # and warn the user. # TODO: I think this is not necessary for new parser!!! Check it if add_reason != '': reason2add = True # casalog.post('Replaced blanks with underscores in cmdreason', 'WARN') # Append to a file if outfile != '': if append: # casalog.post('Will append commands to the file '+outfile) ffout = open(outfile, 'a') else: # casalog.post('Will save commands to the file '+outfile) os.system('rm -f '+outfile) ffout = open(outfile, 'w') try: for key in flagdict.keys(): cmdline = "" cmddict = flagdict[key]['command'] reason = flagdict[key]['reason'] # Add new reason or replace old one with new reason if reason2add: reason = add_reason if reason != '': cmddict['reason'] = reason for k,v in cmddict.items(): cmdstr = "" if isinstance(v, str): # Add quotes to string values if v.count("'") > 0: v = v.strip("'") if v.count('"') > 0: v = v.strip("'") cmdstr = "'"+v+"'" cmdline = cmdline + k + '=' + str(cmdstr) + ' ' else: cmdline = cmdline + k + '=' + str(v) + ' ' # Save to output file ffout.write('%s\n' %cmdline.rstrip()) ffout.close() except: raise Exception('Error writing/appending lines to file ' \ + outfile) ffout.close() return False else: # Append new commands to existing FLAG_CMD table # casalog.post('Saving commands to the FLAG_CMD table') cmdlist = [] reasonlist = [] for key in flagdict.keys(): cmdline = "" reason = "" cmddict = flagdict[key]['command'] # Do not save reason in the COMMAND column if 'reason' in cmddict: cmddict.pop('reason') # Summary cmds should not go to FLAG_CMD if 'mode' in cmddict and cmddict['mode'] == 'summary': casalog.post("Commands with mode='summary' are not allowed in the FLAG_CMD table", 'WARN') continue # Add to REASON column the user input reason if requested reason = flagdict[key]['reason'] if reason2add: reason = add_reason for k,v in cmddict.items(): cmdstr = "" if isinstance(v, str): # Add quotes to string values if v.count("'") > 0: v = v.strip("'") if v.count('"') > 0: v = v.strip("'") cmdstr = "'"+v+"'" cmdline = cmdline + k + '=' + str(cmdstr) + ' ' else: cmdline = cmdline + k + '=' + str(v) + ' ' cmdline = cmdline.rstrip() cmdlist.append(cmdline) reasonlist.append(reason) # Number of cmds to add to FLAG_CMD Nadd = cmdlist.__len__() # Fill in other columns appliedlist = [applied for i in range(Nadd)] timelist = [0.0]*Nadd otherlist = [0]*Nadd typelist = ['']*Nadd mstable = msfile + '/FLAG_CMD' try: tblocal.open(mstable, nomodify=False) except: raise Exception('Error opening FLAG_CMD table '+mstable) nrows = int(tblocal.nrows()) casalog.post('There are ' + str(nrows)+ ' rows already in FLAG_CMD') # Add blank rows first tblocal.addrows(Nadd) # Fill in the columns tblocal.putcol('APPLIED', numpy.array(appliedlist), startrow=nrows,nrow=Nadd) tblocal.putcol('COMMAND', numpy.array(cmdlist), startrow=nrows, nrow=Nadd) tblocal.putcol('INTERVAL', numpy.array(timelist), startrow=nrows,nrow=Nadd) tblocal.putcol('LEVEL', numpy.array(otherlist), startrow=nrows,nrow=Nadd) tblocal.putcol('REASON', numpy.array(reasonlist),startrow=nrows, nrow=Nadd) tblocal.putcol('SEVERITY', numpy.array(otherlist), startrow=nrows,nrow=Nadd) tblocal.putcol('TIME', numpy.array(timelist), startrow=nrows,nrow=Nadd) tblocal.putcol('TYPE', numpy.array(typelist), startrow=nrows,nrow=Nadd) newrows = int(tblocal.nrows()) newrows -= nrows if newrows == 0: casalog.post('Did not save any rows to FLAG_CMD') else: casalog.post('Saved ' + str(newrows) + ' rows to FLAG_CMD of %s'%msfile) tblocal.close() return True def parseRFlagOutputFromSummary(mode,summary_stats_list, flagcmd): """ Function to pull out 'rflag' output from the long dictionary, and (1) write the output files with thresholds. If the filename is specified, use it. If filename is not specified, make one up. (2) modify entries in 'cmdline' so that it is ready for savepars. This is to ensure that 'savepars' saves the contents of the threshold-files and not just the file-names. It has to save it in the form that flagdata accepts inline : e.g. timedev=[[1,10,0.1],[1,11,0.07]] . This way, the user need not keep track of threshold text files if they use 'savepars' with action='apply'. """ # The careful checks are especially for MMS which can produce empty selections/results from # individual sub-MSs. if summary_stats_list and type(summary_stats_list) is dict and 'nreport' in summary_stats_list: nreps = summary_stats_list['nreport'] for rep in range(0,nreps): repname = 'report'+str(rep) if summary_stats_list[repname]['type'] == "rflag": # Pull out the rflag threshold dictionary. This has a 'name' in it. rflag_thresholds = summary_stats_list[repname] # Get the rflag id, to later construct a 'name' from to match the above. rflagid = 0 if mode=='list': rflagid = int( rflag_thresholds['name'].replace('Rflag_','') ) # Go through the flagcmd list, to find the 'rflags'..... for key in flagcmd.keys(): # cmdline is a dictionary with flag commands cmdline = flagcmd[key]['command'] if 'mode' in cmdline and cmdline['mode'] == 'rflag': # Check for match between input flagcmd and output threshold, via the rflag id if(key == rflagid): # If timedev,freqdev are missing from cmdline, add empty ones. if not 'timedev' in cmdline: # aah. don't confuse it with timedevscale cmdline['timedev'] = [] if not 'freqdev' in cmdline: cmdline['freqdev'] = [] # Write RFlag thresholds to these file names. newtimedev,newfreqdev = writeRFlagThresholdFile(rflag_thresholds, cmdline['timedev'], cmdline['freqdev'], rflagid) ## VERIFY for parser # Modify the flagcmd string, so that savepars sees the contents of the file if( isinstance(cmdline['timedev'], list)): cmdline['timedev'] = newtimedev if( isinstance(cmdline['freqdev'],list)): cmdline['freqdev'] = newfreqdev # Remove writeflags from the cmd to prevent it from going into savepars cmdline.pop('writeflags') def save_rflag_consolidated_files(mode, action, cons_dict, opts_dict, inpfile): """ Utility for RFlag when running in parallel/MMS. Does parseRFlagOutputFromSummary() on the consolidated (on the client process) - only when needed for rflag mode (or list mode when the list of commands contains some RFlag commands). :param mode: mode parameter as given to the from casatasks.flagdata import flagdata task :param action: action parameter as given to the flagdata task :param cons_dict: consolidated dictionary of reports from (RFlag) flagdata commands :param opts_dict: dictionary with the options needed for parseRFlagOutputFromSummary() (timedev, freqdev and writeflags, from the task input parameters) :param inpfile: inpfile parameter as given to the flagdata task """ if mode in ['rflag', 'list'] and action != 'apply': import pprint casalog.post('Saving RFlag return dictionary: {0}'. format(pprint.pformat(cons_dict)), 'INFO') # Prepare the list of commands if mode == 'list': cmd_list = get_flag_cmd_list(inpfile) else: # need only the relevant fields to save the output files in # parseRFlagOutputFromSummary() cmd_list = {0: {'command': dict({'mode': 'rflag'}, **opts_dict)}} parseRFlagOutputFromSummary(mode, cons_dict, cmd_list) # Not used at the moment!!!!!!!!!!! def readFlagCmdTable(msfile, rows=[], applied=True, reason='any'): '''Read flag commands from rows of the FLAG_CMD table of msfile If useapplied=False then include only rows with APPLIED=False If myreason is anything other than 'any', then select on that''' # # Return flagcmd structure: # # The flagcmd key is the integer row number from FLAG_CMD # # Dictionary structure: # key : 'id' (string) # 'mode' (string) flag mode '','clip','shadow','quack' # 'antenna' (string) # 'timerange' (string) # 'reason' (string) # 'time' (float) in mjd seconds # 'interval' (float) in mjd seconds # 'cmd' (string) string (for COMMAND col in FLAG_CMD) # 'type' (string) 'FLAG' / 'UNFLAG' # 'applied' (bool) set to True here on read-in # 'level' (int) set to 0 here on read-in # 'severity' (int) set to 0 here on read-in # Open and read columns from FLAG_CMD mstable = msfile + '/FLAG_CMD' # Note, tblocal.getcol doesn't allow random row access, read all try: tblocal.open(mstable) f_time = tblocal.getcol('TIME') f_interval = tblocal.getcol('INTERVAL') f_type = tblocal.getcol('TYPE') f_reas = tblocal.getcol('REASON') f_level = tblocal.getcol('LEVEL') f_severity = tblocal.getcol('SEVERITY') f_applied = tblocal.getcol('APPLIED') f_cmd = tblocal.getcol('COMMAND') tblocal.close() except: casalog.post('Error reading table ' + mstable, 'ERROR') raise Exception nrows = f_time.__len__() if nrows == 0: casalog.post('FLAG_CMD table in %s is empty, no flags extracted' % msfile, 'ERROR') return {} rowlist = [] # Select by rows if rows.__len__() > 0: rowlist = rows else: rowlist = range(nrows) # Select by APPLIED if not applied: rowl = [] for i in rowlist: if not f_applied[i]: rowl.append(i) rowlist = rowl # Select by REASON myreaslist = [] if type(reason) == str: if reason != 'any': myreaslist.append(reason) elif type(reason) == list: myreaslist = reason else: casalog.post('Cannot read reason; it contains unknown variable types' , 'ERROR') return if myreaslist.__len__() > 0: rowl = [] for i in rowlist: if myreaslist.count(f_reas[i]) > 0: rowl.append(i) rowlist = rowl Ncmds = 0 myflagcmd = {} # Define the parser myParser = Parser(' ','=') for i in rowlist: parsed = {} flagd = {} cmd = f_cmd[i] if cmd == '': continue # Parse to a dictionary parsed = myParser.parse2Dictionary(cmd) # This is the original row number in FLAG_CMD flagd['ROW'] = i flagd['COMMAND'] = parsed flagd['REASON'] = f_reas[i] flagd['APPLIED'] = f_applied[i] flagd['TIME'] = f_time[i] flagd['INTERVAL'] = f_interval[i] flagd['TYPE'] = f_type[i] flagd['LEVEL'] = f_level[i] flagd['SEVERITY'] = f_severity[i] myflagcmd[Ncmds] = flagd Ncmds += 1 casalog.post('Read ' + str(Ncmds) + ' rows from FLAG_CMD table in ' + msfile) return myflagcmd def plotFlagCommands(myflags,plotname,t1sdata,t2sdata,): # # Function to plot flagging dictionary # Adapted from J.Marvil # Updated STM v4.1 2011-11-02 to handle ALMA flags # Updated STM v4.2 2012-02-16 trim flag times to data times # Updated STM v4.2 2012-04-10 bug fix in trim flag times to data times # Updated STM v4.2 2012-04-10 messages to logger try: import pylab as pl except ImportError as exc: raise ImportError('Failed to load pylab, required in plotFlagCommands: ', exc) # list of supported colors (in order) colorlist = [ 'red', 'blue', 'green', 'black', 'cyan', 'magenta', 'yellow', 'orange', ] ncolors = colorlist.__len__() # get list of flag keys keylist = myflags.keys() # get lists of antennas and reasons # make plotting dictionary myants = [] myreas = [] plotflag = {} ipf = 0 for key in keylist: antstr = '' reastr = '' timstr = '' if 'antenna' in myflags[key]: antstr = myflags[key]['antenna'] if 'reason' in myflags[key]: reastr = myflags[key]['reason'] if 'timerange' in myflags[key]: timstr = myflags[key]['timerange'] if antstr != '': # flags that have antenna specified antlist = antstr.split(',') nantlist = antlist.__len__() else: # Special antlist = ['All'] nantlist = 1 # realist = reastr.split(',') nrealist = realist.__len__() # timlist = timstr.split(',') ntimlist = timlist.__len__() # # Break these into nants x ntimes flags # Trick is assigning multiple reasons # Normal cases: # A. One reason, single/multiple antennas x times # B. Multiple reasons=times, single/multiple antenna(s) # C. Single time, multiple antennas/reasons # D. Multiple reasons, no way to correspond with times # timmin = 1.0E11 timmax = 0.0 if nrealist == 1: # simplest case, single reason reas = realist[0] if reas == '': reas = 'Unknown' if myreas.count(reas) == 0: myreas.append(reas) for ia in range(nantlist): ant = antlist[ia] if myants.count(ant) == 0: myants.append(ant) for it in range(ntimlist): times = timlist[it] plotflag[ipf] = {} plotflag[ipf]['antenna'] = ant plotflag[ipf]['reason'] = reas plotflag[ipf]['timerange'] = times plotflag[ipf]['show'] = True ipf += 1 elif nrealist == ntimlist: # corresponding reasons and times for ia in range(nantlist): ant = antlist[ia] if myants.count(ant) == 0: myants.append(ant) for it in range(ntimlist): times = timlist[it] reas = realist[it] if reas == '': reas = 'Unknown' if myreas.count(reas) == 0: myreas.append(reas) plotflag[ipf] = {} plotflag[ipf]['antenna'] = ant plotflag[ipf]['reason'] = reas plotflag[ipf]['timerange'] = times plotflag[ipf]['show'] = True ipf += 1 else: # no correspondence between multiple reasons and ants/times # assign reason 'Miscellaneous' reas = 'Miscellaneous' if myreas.count(reas) == 0: myreas.append(reas) for ia in range(nantlist): ant = antlist[ia] if myants.count(ant) == 0: myants.append(ant) for it in range(ntimlist): times = timlist[it] plotflag[ipf] = {} plotflag[ipf]['antenna'] = ant plotflag[ipf]['reason'] = reas plotflag[ipf]['timerange'] = times plotflag[ipf]['show'] = True ipf += 1 myants.sort() nants = myants.__len__() nreas = myreas.__len__() casalog.post('Found ' + str(nreas) + ' reasons to plot for ' + str(nants) + ' antennas') npf = ipf casalog.post('Found ' + str(npf) + ' total flag ranges to plot') # sort out times for ipf in range(npf): times = plotflag[ipf]['timerange'] if times != '': if times.count('~') > 0: t1 = times[:times.find('~')] t2 = times[times.find('~') + 1:] else: t1 = times t2 = t1 (t1s, t2s) = (qalocal.convert(t1, 's')['value'], qalocal.convert(t2, 's')['value']) plotflag[ipf]['t1s'] = t1s plotflag[ipf]['t2s'] = t2s if t1s < timmin: timmin = t1s if t2s > timmax: timmax = t2s # min,max times q1 = qalocal.quantity(timmin, 's') time1 = qalocal.time(q1, form='ymd', prec=9)[0] q2 = qalocal.quantity(timmax, 's') time2 = qalocal.time(q2, form='ymd', prec=9)[0] casalog.post('Found flag times from ' + time1 + ' to ' + time2) # sort out blank times for ipf in range(npf): times = plotflag[ipf]['timerange'] if times == '': if t2sdata >= t1sdata > 0: plotflag[ipf]['t1s'] = t1sdata plotflag[ipf]['t2s'] = t2sdata else: plotflag[ipf]['t1s'] = timmin plotflag[ipf]['t2s'] = timmax # if flag times are beyond range of data, trim them # Added STM 2012-02-16, fixed STM 2012-04-10 ndropped = 0 if t2sdata >= t1sdata > 0 and (timmin < t1sdata or timmax > t2sdata): # min,max data times q1 = qalocal.quantity(t1sdata, 's') tdata1 = qalocal.time(q1, form='ymd', prec=9)[0] q2 = qalocal.quantity(t2sdata, 's') tdata2 = qalocal.time(q2, form='ymd', prec=9)[0] casalog.post('WARNING: Trimming flag times to data limits ' + tdata1 + ' to ' + tdata2) for ipf in range(npf): t1s = plotflag[ipf]['t1s'] t2s = plotflag[ipf]['t2s'] if t1s < t1sdata: if t2s >= t1sdata: # truncate to t1sdata plotflag[ipf]['t1s'] = t1sdata else: # entirely outside data range, do not plot plotflag[ipf]['show'] = False ndropped += 1 if t2s > t2sdata: if t1s <= t2sdata: # truncate to t2sdata plotflag[ipf]['t2s'] = t2sdata else: # entirely outside data range, do not plot plotflag[ipf]['show'] = False ndropped += 1 if ndropped > 0: casalog.post('WARNING: Trimming dropped ' + str(ndropped) + ' flags entirely') # make reason dictionary with mapping of colors and offsets (-0.3 to 0.3) readict = {} reakeys = [] if nreas > ncolors: for i in range(nreas): reas = myreas[i] readict[reas] = {} if i < ncolors - 1: colr = colorlist[i] readict[reas]['color'] = colr readict[reas]['index'] = i offs = 0.3 - float(i) * 0.6 / float(ncolors - 1) readict[reas]['offset'] = offs reakeys.append(reas) else: colr = colorlist[ncolors - 1] readict[reas]['color'] = colr readict[reas]['index'] = ncolors - 1 readict[reas]['offset'] = -0.3 reakeys.append('Other') readict['Other'] = {} readict['Other']['color'] = colorlist[ncolors - 1] readict['Other']['index'] = ncolors - 1 readict['Other']['offset'] = -0.3 else: for i in range(nreas): reas = myreas[i] reakeys.append(reas) colr = colorlist[i] offs = 0.3 - float(i) * 0.6 / float(ncolors - 1) readict[reas] = {} readict[reas]['color'] = colr readict[reas]['index'] = i readict[reas]['offset'] = offs nlegend = reakeys.__len__() casalog.post('Will plot ' + str(nlegend) + ' reasons in legend') antind = 0 if plotname == '': pl.ion() else: pl.ioff() f1 = pl.figure() ax1 = f1.add_axes([.15, .1, .75, .85]) # ax1.set_ylabel('antenna') # ax1.set_xlabel('time') # badflags=[] nplotted = 0 for thisant in myants: antind += 1 for ipf in range(npf): if plotflag[ipf]['show'] and plotflag[ipf]['antenna'] \ == thisant: # plot this flag thisReason = plotflag[ipf]['reason'] thisColor = readict[thisReason]['color'] thisOffset = readict[thisReason]['offset'] t1s = plotflag[ipf]['t1s'] t2s = plotflag[ipf]['t2s'] ax1.plot([t1s, t2s], [antind + thisOffset, antind + thisOffset], color=thisColor, lw=2, alpha=.7) nplotted += 1 casalog.post('Plotted ' + str(nplotted) + ' flags') myXlim = ax1.get_xlim() myXrange = myXlim[1] - myXlim[0] # Pad the time axis? PadTime = 0.050000000000000003 if PadTime > 0: xPad = PadTime * myXrange x0 = myXlim[0] - xPad x1 = myXlim[1] + xPad ax1.set_xlim(x0, x1) myXrange = x1 - x0 else: # casalog.post(' Rescaled x axis') x0 = myXlim[0] x1 = myXlim[1] legendFontSize = 12 myYupper = nants + nlegend + 1.5 # place legend text i = nants x = x0 + 0.050000000000000003 * myXrange for reas in reakeys: i += 1 colr = readict[reas]['color'] ax1.text(x, i, reas, color=colr, size=legendFontSize) ax1.set_ylim([0, myYupper]) ax1.set_yticks(range(1, len(myants) + 1)) ax1.set_yticklabels(myants) # casalog.post(' Relabled y axis') nxticks = 3 ax1.set_xticks(pl.linspace(myXlim[0], myXlim[1], nxticks)) mytime = [] myTimestr = [] for itim in range(nxticks): time = myXlim[0] + (myXlim[1] - myXlim[0]) * float(itim) \ / float(nxticks - 1) mytime.append(time) q1 = qalocal.quantity(time, 's') time1 = qalocal.time(q1, form='ymd', prec=9)[0] if itim > 0: time1s = time1[11:] else: time1s = time1 myTimestr.append(time1s) ax1.set_xticklabels(myTimestr) # casalog.post(myTimestr) if plotname == '': pl.draw() else: pl.savefig(plotname, dpi=150) return def evaluateParameters(pardict): '''Evaluate the correct types in a string with parameters Keyword arguments: pardict -- a dictionary with flag commands as returned by parseDictionary() It evaluates each value to its correct Python type. ''' cmddict = OrderedDict() for key,val in pardict.items(): newval = None if val.startswith('['): newval = ast.literal_eval(val) elif val.startswith('{'): newval = eval(val) elif val in ['True', 'False']: if eval(val) in [True, False]: newval = eval(val) if newval is None: try: # Try int int(val) newval = int(val) except: # Try float try: float(val) newval = float(val) except: # Try string if val.count("'") > 0: val = val.strip("'") elif val.count('"') > 0: val = val.strip('"') # CAS-6553 cannot have only one quote. remove it if val.count("'") == 1: val = val.replace("'", '') elif val.count('"') == 1: val = val.replace('"', '') newval = str(val).strip() cmddict[key] = newval return cmddict def evaluateFlagParameters(pardict, pars): """Check if a flagdata parameter dictionary is valid Keyword arguments: pardict -- a dictionary with flag commands as returned by parseDictionary() pars -- dictionary of flagdata's parameters (locals() inside flagdata task) It checks if the parameter exists in the reference tuple. It checks if the type of the parameter value matches the type in the reference for that parameter. It raises an exception if any parameter or type of value do not match. """ from casatasks import flagdata # Make a deepcopy of flagdata parameters dictionary for modification fpars = copy.deepcopy(pars) # Get the defaults of each parameter for par in fpars.keys(): fpars[par] = get_task_arg_default(flagdata, par) # Define the parameters that don't go in an input list in flagdata removepars = ['vis','inpfile','flagbackup','tbuff','cmdreason','savepars','outfile', 'display','action'] # Remove the parameters we don't need to evaluate for par in removepars: if par in fpars: fpars.pop(par) # Define the parameters that have variant type in flagdata dup_pars = ['ntime','observation','addantenna','timedev','timedev','freqdev','freqdev','quackinterval'] dup_types = [0.0,0,{},[],0.0,[],0.0,0] # Create a tuple from flagdata's parameters ftup = fpars.items() # Create a list of the tuples flist = list(ftup) # Append the duplicated types for d in range(len(dup_pars)): # Create a tuple and append to list dpar = (dup_pars[d],dup_types[d]) flist.append(dpar) # Create a tuple of the reference dictionary reftup = tuple(flist) # Reference parameters for flagdata task # reftup = (('field','string'),('spw','string'),('antenna','string'),('timerange','string'),('correlation','string'), # ('scan','string'),('intent','string'),('array','string'),('uvrange','string'), # ('observation','string'), # ('observation',0), # ('feed','string'), # ('mode','string'), # ('reason','string'), # # manual # ('autocorr', True), # # clip # ('datacolumn','string'), # ('clipminmax',[]), # ('clipoutside',True), # ('channelavg',False), # ('clipzeros',False), # # elevation # ('lowerlimit',0.0), # ('upperlimit',90.0), # # shadow # ('tolerance',0.0), # ('addantenna','string'), # ('addantenna',{}), # # quack # ('quackinterval',0.0), # ('quackinterval',0), # ('quackmode','string'), # ('quackincrement',False), # # tfcrop # ('ntime','string'), # ('ntime',0.0), # ('combinescans',False), # ('timecutoff',4.0), # ('freqcutoff',3.0), # ('timefit','string'), # ('freqfit','string'), # ('maxnpieces',7), # ('flagdimension','string'), # ('usewindowstats','string'), # ('halfwin',1), # ('extendflags',True), # # rflag # ('winsize',3), # ('timedev','string'), # ('timedev',[]), # ('timedev',0.0), # ('freqdev','string'), # ('freqdev',[]), # ('freqdev',0.0), # ('timedevscale',5.0), # ('freqdevscale',5.0), # ('spectralmax',1000000.0), # ('spectralmin',0.0), # # extend # ('extendpols',True), # ('growtime',50.0), # ('growfreq',50.0), # ('growaround',False), # ('flagneartime',False), # ('flagnearfreq',False), # # summary # ('minrel',0.0), # ('maxrel',1.0), # ('minabs',0), # ('maxabs',1), # ('spwchan',False), # ('spwcorr',False), # ('basecnt',False), # ('name','string')) reference = defaultdict(list) for k, v in reftup: reference[k].append(v) refkeys = reference.keys() # Check the user dictionary against the reference count = 0 for idx in pardict.keys(): mydict = pardict[idx]['command'] count += 1 for key,val in mydict.items(): if key not in refkeys: raise ValueError('Parameter \'%s\' in row=%s is not a valid flagdata parameter'%(key,idx)) # reference[key] is always a list refval = reference[key] vtypes = "" match = False for tt in refval: vtypes = vtypes + str(type(tt)) + ',' if type(val) == type(tt): # type matches match = True if not match: raise(ValueError,'Parameter %s=%s in row=%s has wrong type.\nSupported types are: %s.'%(key,val,idx,vtypes.rstrip(','))) casalog.post('Evaluated %s rows of dictionary'%count,'DEBUG1') return True def evaluateNumpyType(elem): '''Evaluate if an element is of numpy type. Cast it to the corresponding Python type and return the casted value''' import numpy as np val = None if(isinstance(elem,np.int) or isinstance(elem,np.int8) or isinstance(elem,np.int16) or isinstance(elem,np.int32) or isinstance(elem,np.int64)): val = int(elem) elif(isinstance(elem,np.float) or isinstance(elem,np.float16) or isinstance(elem,np.float32) or isinstance(elem,np.float64) or isinstance(elem,np.float128)): val = float(elem) elif(isinstance(elem,np.double)): val = float(elem) else: # it is none of the above numpy types val = elem # return the casted element return val def parseXML(sdmfile, mytbuff): ''' # readflagxml: reads Antenna.xml and Flag.xml SDM tables and parses # into returned dictionary as flag command strings # sdmfile (string) path to SDM containing Antenna.xml and Flag.xml # mytbuff (float) time interval (start and end) padding (seconds) # # Usage: myflags = readflagxml(sdmfile,tbuff) # # Dictionary structure: # fid : 'id' (string) # 'mode' (string) flag mode ('online') # 'antenna' (string) # 'timerange' (string) # 'reason' (string) # 'time' (float) in mjd seconds # 'interval' (float) in mjd seconds # 'command' (string) string (for COMMAND col in FLAG_CMD) # 'type' (string) 'FLAG' / 'UNFLAG' # 'applied' (bool) set to True here on read-in # 'level' (int) set to 0 here on read-in # 'severity' (int) set to 0 here on read-in # # Updated STM 2011-11-02 handle new SDM Flag.xml format from ALMA # Updated STM 2012-02-14 handle spectral window indices, names, IDs # Updated STM 2012-02-21 handle polarization types # # Mode to use for spectral window selection in commands: # spwmode = 0 none (flag all spw) # spwmode = 1 use name # spwmode = -1 use index (counting rows in SpectralWindow.xml) # # Mode to use for polarization selection in commands: # polmode = 0 none (flag all pols/corrs) # polmode = 1 use polarization type # # CURRENT DEFAULT: Use spw names, flag pols ''' spwmode = 1 polmode = 1 # try: from xml.dom import minidom except ImportError as exc: raise ImportError('Failed to load xml.dom.minidom, required in parseXML: ', exc) if type(mytbuff) != float: raise RuntimeError('Found incorrect type for tbuff. type(mytbuff): {}'. format(type(mytbuff))) # make sure Flag.xml and Antenna.xml are available (SpectralWindow.xml depends) flagexist = os.access(sdmfile + '/Flag.xml', os.F_OK) antexist = os.access(sdmfile + '/Antenna.xml', os.F_OK) spwexist = os.access(sdmfile + '/SpectralWindow.xml', os.F_OK) if not flagexist: casalog.post('Cannot open ' + sdmfile + '/Flag.xml', 'SEVERE') exit(1) if not antexist: casalog.post('Cannot open ' + sdmfile + '/Antenna.xml', 'SEVERE' ) exit(1) if not spwexist: casalog.post('Cannot open ' + sdmfile + '/SpectralWindow.xml', 'WARN') # construct look-up dictionary of name vs. id from Antenna.xml xmlants = minidom.parse(sdmfile + '/Antenna.xml') antdict = {} rowlist = xmlants.getElementsByTagName('row') for rownode in rowlist: rowname = rownode.getElementsByTagName('name') ant = str(rowname[0].childNodes[0].nodeValue).strip() rowid = rownode.getElementsByTagName('antennaId') # CAS-4532: remove spaces between content and tags antid = str(rowid[0].childNodes[0].nodeValue).strip() antdict[antid] = ant casalog.post('Found ' + str(rowlist.length) + ' rows in Antenna.xml') # construct look-up dictionary of name vs. id from SpectralWindow.xml if spwexist: xmlspws = minidom.parse(sdmfile + '/SpectralWindow.xml') spwdict = {} rowlist = xmlspws.getElementsByTagName('row') ispw = 0 wvrnominal = False for rownode in rowlist: rowid = rownode.getElementsByTagName('spectralWindowId') # CAS-4532: remove spaces between content and tags spwid = str(rowid[0].childNodes[0].nodeValue).strip() spwdict[spwid] = {} spwdict[spwid]['index'] = ispw if rownode.getElementsByTagName('name'): rowname = rownode.getElementsByTagName('name') spwname = str(rowname[0].childNodes[0].nodeValue).strip() if spwname == 'WVR#NOMINAL': wvrnominal = True spwdict[spwid]['name'] = spwname else: spwmode = -1 # rowid = rownode.getElementsByTagName('spectralWindowId') # spwid = str(rowid[0].childNodes[0].nodeValue) # spwdict[spwid] = {} # spwdict[spwid]['index'] = ispw ispw += 1 casalog.post('Found ' + str(rowlist.length) + ' rows in SpectralWindow.xml') # report chosen spw and pol modes if spwmode > 0: casalog.post('Will construct spw flags using names') elif spwmode < 0: casalog.post('Will construct spw flags using table indices') else: casalog.post('') # if polmode == 0: casalog.post('Will not set polarization dependent flags (flag all corrs)' ) else: casalog.post('Will construct polarization flags using polarizationType' ) # now read Flag.xml into dictionary row by row xmlflags = minidom.parse(sdmfile + '/Flag.xml') flagdict = {} rowlist = xmlflags.getElementsByTagName('row') nrows = rowlist.length newsdm = -1 newspw = -1 newpol = -1 for fid in range(nrows): rownode = rowlist[fid] rowfid = rownode.getElementsByTagName('flagId') fidstr = str(rowfid[0].childNodes[0].nodeValue).strip() flagdict[fid] = {} flagdict[fid]['id'] = fidstr rowid = rownode.getElementsByTagName('antennaId') antid = rowid[0].childNodes[0].nodeValue # check if there is a numAntenna specified (new format) rownant = rownode.getElementsByTagName('numAntenna') antname = '' if rownant.__len__() > 0: xid = antid.split() nant = int(rownant[0].childNodes[0].nodeValue) if newsdm < 0: casalog.post('Found numAntenna=' + str(nant) + ' must be a new style SDM') newsdm = 1 # CAS-4698. Flag auto-correlations when there is # only one antenna if nant == 1: aid = xid[2] ana = antdict[aid] if antname == '': antname = ana+'&&*' else: antname += ',' + ana elif nant > 1: for ia in range(nant): aid = xid[2 + ia] ana = antdict[aid] if antname == '': antname = ana else: antname += ',' + ana else: # numAntenna = 0 means flag all antennas antname = '' else: if newsdm < 0: casalog.post('No numAntenna entry found, must be an old style SDM' ) newsdm = 0 nant = 1 aid = antid ana = antdict[aid] antname = ana # start and end times in mjd ns rowstart = rownode.getElementsByTagName('startTime') start = int(rowstart[0].childNodes[0].nodeValue) startmjds = float(start) * 1.0E-9 - mytbuff t = qalocal.quantity(startmjds, 's') starttime = qalocal.time(t, form='ymd', prec=9)[0] rowend = rownode.getElementsByTagName('endTime') end = int(rowend[0].childNodes[0].nodeValue) endmjds = float(end) * 1.0E-9 + mytbuff t = qalocal.quantity(endmjds, 's') endtime = qalocal.time(t, form='ymd', prec=9)[0] # time and interval for FLAG_CMD use times = 0.5 * (startmjds + endmjds) intervs = endmjds - startmjds flagdict[fid]['time'] = times flagdict[fid]['interval'] = intervs # reasons rowreason = rownode.getElementsByTagName('reason') reas = str(rowreason[0].childNodes[0].nodeValue) # Replace any white space with underscores reason = reas.replace(' ','_') # NEW SDM ADDITIONS 2011-11-01 rownspw = rownode.getElementsByTagName('numSpectralWindow') spwstring = '' if spwmode != 0 and rownspw.__len__() > 0: nspw = int(rownspw[0].childNodes[0].nodeValue) # has a new-style spw specification if newspw < 0: if not spwexist: casalog.post('Cannot open ' + sdmfile + '/SpectralWindow.xml', 'SEVERE') exit(1) casalog.post('Found numSpectralWindow=' + str(nspw) + ' must be a new style SDM') newspw = 1 if nspw > 0: rowspwid = rownode.getElementsByTagName('spectralWindowId') spwids = rowspwid[0].childNodes[0].nodeValue xspw = spwids.split() for isp in range(nspw): spid = str(xspw[2 + isp]) if spwmode > 0: spstr =spwdict[spid]['name'] # Skip the WVR#Antenna_* if WVR#NOMINAL is present if spstr.__contains__('WVR#Antenna') and wvrnominal==True: continue if spwstring == '': spwstring = '"'+spstr+'"' else: spwstring += ',' + '"'+spstr+'"' else: spstr = (spwdict[spid]['index']) if spwstring == '': spwstring = spstr else: spwstring += ',' + spstr polstring = '' rownpol = rownode.getElementsByTagName('numPolarizationType') if polmode != 0 and rownpol.__len__() > 0: npol = int(rownpol[0].childNodes[0].nodeValue) # has a new-style pol specification if newpol < 0: casalog.post('Found numPolarizationType=' + str(npol) + ' must be a new style SDM') newpol = 1 if npol > 0: rowpolid = \ rownode.getElementsByTagName('polarizationType') polids = rowpolid[0].childNodes[0].nodeValue xpol = polids.split() for ipol in range(npol): polid = str(xpol[2 + ipol]) if polstring == '': polstring = polid else: polstring += ',' + polid # # Construct antenna name and timerange and reason strings flagdict[fid]['antenna'] = antname timestr = starttime + '~' + endtime flagdict[fid]['timerange'] = timestr flagdict[fid]['reason'] = reason # Construct command strings (per input flag) cmddict = OrderedDict() # cmd = "antenna='" + antname + "' timerange='" + timestr + "'" # cmddict['mode'] = 'manual' cmddict['antenna'] = antname cmddict['timerange'] = timestr if spwstring != '': # cmd += " spw='" + spwstring + "'" flagdict[fid]['spw'] = spwstring cmddict['spw'] = spwstring # if polstring != '': # cmd += " poln='" + polstring + "'" # flagdict[fid]['poln'] = polstring if polstring != '': # Write the poln translation in correlation if polstring.count('R')>0: if polstring.count('L')>0: corr = 'RR,RL,LR,LL' else: corr = 'RR,RL,LR' elif polstring.count('L')>0: corr = 'LL,LR,RL' elif polstring.count('X')>0: if polstring.count('Y')>0: corr = 'XX,XY,YX,YY' else: corr = 'XX,XY,YX' elif polstring.count('Y')>0: corr = 'YY,YX,XY' # cmd += " correlation='" + corr + "'" cmddict['correlation'] = corr # flagdict[fid]['poln'] = polstring # flagdict[fid]['command'] = cmd flagdict[fid]['command'] = cmddict # flagdict[fid]['type'] = 'FLAG' flagdict[fid]['applied'] = False flagdict[fid]['level'] = 0 flagdict[fid]['severity'] = 0 flagdict[fid]['mode'] = 'online' flags = {} if rowlist.length > 0: flags = flagdict casalog.post('Found ' + str(rowlist.length) + ' flags in Flag.xml') else: casalog.post('No valid flags found in Flag.xml') # return the dictionary for later use return flags # END of NEW PARSER FUNCTIONS ############################################################################################## ############################################################################################## # (CAS-4119): Use absolute paths for input files to ensure that the engines find them def addAbsPath(input_file): '''Read in the lines from an input file input_file --> file in disk with a list of strings per line Re-writes the file changing relative file names to absolute file names ''' output_file = input_file + ".tmp" if (type(input_file) == str) & os.path.exists(input_file): try: input_file_id = open(input_file, 'r') except: raise Exception('Error opening file ' + input_file) try: output_file_id = open(output_file, 'w') except: output_file_id.close() raise Exception('Error opening tmp file ' + output_file) else: raise Exception('ASCII file not found - please verify the name') # # Parse file try: for line in input_file_id: cmd = line.rstrip() cmd_pars = cmd.split(" ") for par in cmd_pars: if ((par.count("inpfile") > 0) or (par.count("outfile") > 0) or (par.count("addantenna") > 0) or (par.count("timedev") > 0) or (par.count("freqdev") > 0)): par = par.split("=",1) file_local = par[1] if file_local.count("'")>0: file_local = file_local.split("'") file_local = file_local[1] elif file_local.count('"')>0: file_local = file_local.split('"') file_local = file_local[1] else: continue file_abs = os.path.abspath(file_local) cmd = cmd.replace(file_local,file_abs) output_file_id.write(cmd+"\n") except: input_file_id.close() output_file_id.close() raise Exception('Error reading lines from file ' \ + input_file) input_file_id.close() output_file_id.close() # os.rename(output_file, input_file) # Return the temporary file return output_file def makeDict(cmdlist, myreason='any'): '''Make a dictionary compatible with a FLAG_CMD table structure and select by reason if any is given cmdlist --> list of parameters to go into the COMMAND column myreason --> reason to select from Returns a dictionary with the the selected rows with the following structure and default values: flagd['row'] = flagd['antenna'] = flagd['timerange'] = flagd['applied'] = False flagd['command'] = '' flagd['interval'] = 0.0 flagd['level'] = 0 flagd['reason'] = '' flagd['severity'] = 0 flagd['time'] = 0.0 flagd['type'] = 'FLAG' ''' if cmdlist.__len__() == 0: raise Exception('Empty list of commands') # select by these reasons myreaslist = [] if type(myreason) == str: if myreason != 'any': myreaslist.append(myreason) elif type(myreason) == list: myreaslist = myreason else: casalog.post('Cannot read reason; it contains unknown variable types', 'ERROR') return if debug: casalog.post("reason in selection: {}".format(myreaslist)) # List of reasons in input file nrows = cmdlist.__len__() # If any selection was requested reaslist = [] for i in range(nrows): command = cmdlist[i] if command.__contains__('reason'): reaslist.append(getReason(command)) else: # so that cmdlist and reaslist have the same sizes reaslist.append('NoReasonToMatch') if debug: casalog.post("reason in input: {}".format(reaslist)) # Defaults for columns applied = False interval = 0.0 level = 0 severity = 0 coltime = 0.0 coltype = 'FLAG' antenna = '' timerange = '' myflagd = {} ncmds = 0 rowlist = range(nrows) # If select by reason is requested if myreaslist.__len__() > 0: rowl = [] # If selection matches what is in input line for j in range(reaslist.__len__()): if myreaslist.count(reaslist[j]) > 0: rowl.append(j) rowlist = rowl try: for i in rowlist: flagd = {} reason = '' command = cmdlist[i] if reaslist[i] != 'NoReasonToMatch': reason = reaslist[i] # Skip comment lines if command.startswith('#'): continue if command == '': casalog.post('Ignoring empty command line', 'WARN') continue if command.__contains__('summary'): casalog.post('Mode summary is not allowed in list operation', 'WARN') continue # If shadow, remove the addantenna dictionary if command.__contains__('shadow') and command.__contains__('addantenna'): i0 = command.rfind('addantenna') if command[i0+11] == '{': # It is a dictionary. Remove it from line i1 = command.rfind('}') antpar = command[i0+11:i1+1] temp = command[i0:i1+1] newcmd = command.replace(temp,'') antpardict = convertStringToDict(antpar) flagd['addantenna'] = antpardict command = newcmd # CAS-5180: Get antenna and timerange to use when action='plot' keyvlist = command.split() if keyvlist.__len__() > 0: for keyv in keyvlist: # Take care of '=' signs in values try: (xkey, val) = keyv.split('=',1) except: casalog.post('Error: not key=value pair for ' + keyv, 'ERROR') break xval = val if xval.count("'") > 0: xval = xval.strip("'") if xval.count('"') > 0: xval = xval.strip('"') if xkey == 'mode': flagd['mode'] = xval elif xkey == 'timerange': timerange = xval elif xkey == 'antenna': antenna = xval flagd['row'] = str(i) flagd['applied'] = applied flagd['reason'] = reason flagd['timerange'] = timerange flagd['antenna'] = antenna # Remove reason from command line command = command.rstrip() newline = purgeParameter(command, 'reason') # Remove any empty parameter (CAS-4015) command = purgeEmptyPars(newline) command = command.strip() flagd['command'] = command flagd['interval'] = interval flagd['level'] = level flagd['severity'] = severity flagd['time'] = coltime flagd['type'] = coltype # Insert into main dictionary myflagd[ncmds] = flagd ncmds += 1 except: raise Exception('Cannot create dictionary') casalog.post(':makeDict::myflagd=%s'%myflagd,'DEBUG') return myflagd def readXML(sdmfile, mytbuff): ''' # readflagxml: reads Antenna.xml and Flag.xml SDM tables and parses # into returned dictionary as flag command strings # sdmfile (string) path to SDM containing Antenna.xml and Flag.xml # mytbuff (float) time interval (start and end) padding (seconds) # # Usage: myflags = readflagxml(sdmfile,tbuff) # # Dictionary structure: # fid : 'id' (string) # 'mode' (string) flag mode ('online') # 'antenna' (string) # 'timerange' (string) # 'reason' (string) # 'time' (float) in mjd seconds # 'interval' (float) in mjd seconds # 'command' (string) string (for COMMAND col in FLAG_CMD) # 'type' (string) 'FLAG' / 'UNFLAG' # 'applied' (bool) set to True here on read-in # 'level' (int) set to 0 here on read-in # 'severity' (int) set to 0 here on read-in # # Updated STM 2011-11-02 handle new SDM Flag.xml format from ALMA # Updated STM 2012-02-14 handle spectral window indices, names, IDs # Updated STM 2012-02-21 handle polarization types # # Mode to use for spectral window selection in commands: # spwmode = 0 none (flag all spw) # spwmode = 1 use name # spwmode = -1 use index (counting rows in SpectralWindow.xml) # # Mode to use for polarization selection in commands: # polmode = 0 none (flag all pols/corrs) # polmode = 1 use polarization type # # CURRENT DEFAULT: Use spw names, flag pols ''' spwmode = 1 polmode = 1 # try: from xml.dom import minidom except ImportError as exc: raise ImportError('failed to load xml.dom.minidom, required in readXML: ', exc) if type(mytbuff) != float: raise RuntimeError('Found incorrect type for tbuff. type(mytbuff): {}'. format(type(mytbuff))) # make sure Flag.xml and Antenna.xml are available (SpectralWindow.xml depends) flagexist = os.access(sdmfile + '/Flag.xml', os.F_OK) antexist = os.access(sdmfile + '/Antenna.xml', os.F_OK) spwexist = os.access(sdmfile + '/SpectralWindow.xml', os.F_OK) if not flagexist: casalog.post('Cannot open ' + sdmfile + '/Flag.xml', 'SEVERE') exit(1) if not antexist: casalog.post('Cannot open ' + sdmfile + '/Antenna.xml', 'SEVERE' ) exit(1) if not spwexist: casalog.post('Cannot open ' + sdmfile + '/SpectralWindow.xml', 'WARN') # construct look-up dictionary of name vs. id from Antenna.xml xmlants = minidom.parse(sdmfile + '/Antenna.xml') antdict = {} rowlist = xmlants.getElementsByTagName('row') for rownode in rowlist: rowname = rownode.getElementsByTagName('name') ant = str(rowname[0].childNodes[0].nodeValue) rowid = rownode.getElementsByTagName('antennaId') # CAS-4532: remove spaces between content and tags antid = str(rowid[0].childNodes[0].nodeValue).strip() antdict[antid] = ant casalog.post('Found ' + str(rowlist.length) + ' antennas in Antenna.xml') # construct look-up dictionary of name vs. id from SpectralWindow.xml if spwexist: xmlspws = minidom.parse(sdmfile + '/SpectralWindow.xml') spwdict = {} rowlist = xmlspws.getElementsByTagName('row') ispw = 0 for rownode in rowlist: rowid = rownode.getElementsByTagName('spectralWindowId') # CAS-4532: remove spaces between content and tags spwid = str(rowid[0].childNodes[0].nodeValue).strip() spwdict[spwid] = {} spwdict[spwid]['index'] = ispw # SMC: 6/3/2012 ALMA SDM does not have name if rownode.getElementsByTagName('name'): rowname = rownode.getElementsByTagName('name') spw = str(rowname[0].childNodes[0].nodeValue) spwdict[spwid]['name'] = spw else: spwmode = -1 # rowid = rownode.getElementsByTagName('spectralWindowId') # spwid = str(rowid[0].childNodes[0].nodeValue) # spwdict[spwid] = {} # spwdict[spwid]['index'] = ispw ispw += 1 casalog.post('Found ' + str(rowlist.length) + ' spw in SpectralWindow.xml') # report chosen spw and pol modes if spwmode > 0: casalog.post('Will construct spw flags using names') elif spwmode < 0: casalog.post('Will construct spw flags using table indices') else: casalog.post('') # if polmode == 0: casalog.post('Will not set polarization dependent flags (flag all corrs)' ) else: casalog.post('Will construct polarization flags using polarizationType' ) # now read Flag.xml into dictionary row by row xmlflags = minidom.parse(sdmfile + '/Flag.xml') flagdict = {} rowlist = xmlflags.getElementsByTagName('row') nrows = rowlist.length newsdm = -1 newspw = -1 newpol = -1 for fid in range(nrows): rownode = rowlist[fid] rowfid = rownode.getElementsByTagName('flagId') fidstr = str(rowfid[0].childNodes[0].nodeValue) flagdict[fid] = {} flagdict[fid]['id'] = fidstr rowid = rownode.getElementsByTagName('antennaId') antid = rowid[0].childNodes[0].nodeValue # check if there is a numAntenna specified (new format) rownant = rownode.getElementsByTagName('numAntenna') antname = '' if rownant.__len__() > 0: xid = antid.split() nant = int(rownant[0].childNodes[0].nodeValue) if newsdm < 0: casalog.post('Found numAntenna=' + str(nant) + ' must be a new style SDM') newsdm = 1 # CAS-4698. Flag auto-correlations when there is # only one antenna if nant == 1: aid = xid[2] ana = antdict[aid] if antname == '': antname = ana+'&&*' else: antname += ',' + ana elif nant > 1: for ia in range(nant): aid = xid[2 + ia] ana = antdict[aid] if antname == '': antname = ana else: antname += ',' + ana else: # numAntenna = 0 means flag all antennas antname = '' else: if newsdm < 0: casalog.post('No numAntenna entry found, must be an old style SDM' ) newsdm = 0 nant = 1 aid = antid ana = antdict[aid] antname = ana # start and end times in mjd ns rowstart = rownode.getElementsByTagName('startTime') start = int(rowstart[0].childNodes[0].nodeValue) startmjds = float(start) * 1.0E-9 - mytbuff t = qalocal.quantity(startmjds, 's') starttime = qalocal.time(t, form='ymd', prec=9)[0] rowend = rownode.getElementsByTagName('endTime') end = int(rowend[0].childNodes[0].nodeValue) endmjds = float(end) * 1.0E-9 + mytbuff t = qalocal.quantity(endmjds, 's') endtime = qalocal.time(t, form='ymd', prec=9)[0] # time and interval for FLAG_CMD use times = 0.5 * (startmjds + endmjds) intervs = endmjds - startmjds flagdict[fid]['time'] = times flagdict[fid]['interval'] = intervs # reasons rowreason = rownode.getElementsByTagName('reason') reas = str(rowreason[0].childNodes[0].nodeValue) # Replace any white space with underscores reason = reas.replace(' ','_') # NEW SDM ADDITIONS 2011-11-01 rownspw = rownode.getElementsByTagName('numSpectralWindow') spwstring = '' if spwmode != 0 and rownspw.__len__() > 0: nspw = int(rownspw[0].childNodes[0].nodeValue) # has a new-style spw specification if newspw < 0: if not spwexist: casalog.post('Cannot open ' + sdmfile + '/SpectralWindow.xml', 'SEVERE') exit(1) casalog.post('Found SpectralWindow=' + str(nspw) + ' must be a new style SDM') newspw = 1 if nspw > 0: rowspwid = \ rownode.getElementsByTagName('spectralWindowId') spwids = rowspwid[0].childNodes[0].nodeValue xspw = spwids.split() for isp in range(nspw): spid = str(xspw[2 + isp]) if spwmode > 0: spstr = spwdict[spid]['name'] else: spstr = str(spwdict[spid]['index']) if spwstring == '': spwstring = spstr else: spwstring += ',' + spstr polstring = '' rownpol = rownode.getElementsByTagName('numPolarizationType') if polmode != 0 and rownpol.__len__() > 0: npol = int(rownpol[0].childNodes[0].nodeValue) # has a new-style pol specification if newpol < 0: casalog.post('Found numPolarizationType=' + str(npol) + ' must be a new style SDM') newpol = 1 if npol > 0: rowpolid = \ rownode.getElementsByTagName('polarizationType') polids = rowpolid[0].childNodes[0].nodeValue xpol = polids.split() for ipol in range(npol): polid = str(xpol[2 + ipol]) if polstring == '': polstring = polid else: polstring += ',' + polid # # Construct antenna name and timerange and reason strings flagdict[fid]['antenna'] = antname timestr = starttime + '~' + endtime flagdict[fid]['timerange'] = timestr flagdict[fid]['reason'] = reason # Construct command strings (per input flag) cmd = "antenna='" + antname + "' timerange='" + timestr + "'" if spwstring != '': cmd += " spw='" + spwstring + "'" flagdict[fid]['spw'] = spwstring # if polstring != '': # cmd += " poln='" + polstring + "'" # flagdict[fid]['poln'] = polstring if polstring != '': # Write the poln translation in correlation if polstring.count('R')>0: if polstring.count('L')>0: corr = 'RR,RL,LR,LL' else: corr = 'RR,RL,LR' elif polstring.count('L')>0: corr = 'LL,LR,RL' elif polstring.count('X')>0: if polstring.count('Y')>0: corr = 'XX,XY,YX,YY' else: corr = 'XX,XY,YX' elif polstring.count('Y')>0: corr = 'YY,YX,XY' cmd += " correlation='" + corr + "'" # flagdict[fid]['poln'] = polstring flagdict[fid]['command'] = cmd # flagdict[fid]['type'] = 'FLAG' flagdict[fid]['applied'] = False flagdict[fid]['level'] = 0 flagdict[fid]['severity'] = 0 flagdict[fid]['mode'] = 'online' flags = {} if rowlist.length > 0: flags = flagdict casalog.post('Found ' + str(rowlist.length) + ' flags in Flag.xml') else: casalog.post('No valid flags found in Flag.xml') # return the dictionary for later use return flags def getUnion(vis, cmddict): '''Get a dictionary of a union of all selection parameters from a list of lines: vis --> MS cmddict --> dictionary of parameters and values (par=val) such as the one returned by makeDict() ''' # Dictionary of parameters to return dicpars = { 'field':'', 'scan':'', 'antenna':'', 'spw':'', 'timerange':'', 'correlation':'', 'intent':'', 'feed':'', 'array':'', 'uvrange':'', 'observation':'' } # Strings for each parameter scans = '' fields = '' ants = '' times = '' corrs = '' ints = '' feeds = '' arrays = '' uvs = '' spws = '' obs = '' nrows = cmddict.keys().__len__() # nrows = cmdlist.__len__() # for i in range(nrows): for k in cmddict.keys(): # cmdline = cmdlist[i] cmdline = cmddict[k]['command'] # Skip if it is a comment line if cmdline.startswith('#'): break # split by white space keyvlist = cmdline.split() if keyvlist.__len__() > 0: # Split by '=' for keyv in keyvlist: (xkey,xval) = keyv.split('=',1) # Remove quotes if type(xval) == str: if xval.count("'") > 0: xval = xval.strip("'") if xval.count('"') > 0: xval = xval.strip('"') # Check which parameter if xkey == "scan": scans += xval + ',' elif xkey == "field": fields += xval + ',' elif xkey == "antenna": ants += xval + ';' elif xkey == "timerange": times += xval + ',' elif xkey == "correlation": corrs += xval + ',' elif xkey == "intent": ints += xval + ',' elif xkey == "feed": feeds += xval + ',' elif xkey == "array": arrays += xval + ',' elif xkey == "uvrange": uvs += xval + ',' elif xkey == "spw": spws += xval + ',' elif xkey == "observation": obs += xval + ',' # Strip out the extra comma at the end scans = scans.rstrip(',') fields = fields.rstrip(',') ants = ants.rstrip(';') times = times.rstrip(',') corrs = corrs.rstrip(',') ints = ints.rstrip(',') feeds = feeds.rstrip(',') arrays = arrays.rstrip(',') uvs = uvs.rstrip(',') spws = spws.rstrip(',') obs = obs.rstrip(',') dicpars['scan'] = scans dicpars['field'] = fields # Antennas are handled better within the framework. dicpars['antenna'] = '' # Times are handled better within the framework. dicpars['timerange'] = '' # Correlations should be handled only by the agents dicpars['correlation'] = '' # CAS-4682, do not create union for intents dicpars['intent'] = '' dicpars['feed'] = feeds dicpars['array'] = arrays dicpars['uvrange'] = uvs dicpars['spw'] = spws dicpars['observation'] = obs # Compress the selection list to reduce MSSelection parsing time. # 'field','spw','antenna' strings in dicpars will be modified in-place. compressSelectionList(vis,dicpars); # Real number of input lines # Get the number of occurrences of each parameter npars = getNumPar(cmddict) nlines = nrows - npars['comment'] # Make the union. for k,v in npars.items(): if k != 'comment': if v < nlines: dicpars[k] = '' uniondic = dicpars.copy() # Remove empty parameters from the dictionary for k,v in dicpars.items(): if v == '': uniondic.pop(k) return uniondic def getNumPar(cmddict): '''Get the number of occurrences of all parameter keys cmdlist --> list of strings with parameters and values ''' # nrows = cmdlist.__len__() # Dictionary of number of occurrences to return npars = { 'field':0, 'scan':0, 'antenna':0, 'spw':0, 'timerange':0, 'correlation':0, 'intent':0, 'feed':0, 'array':0, 'uvrange':0, 'comment':0, 'observation':0 } ci = 0 # count the number of lines with comments (starting with a #) si = 0 # count the number of lines with scan fi = 0 # count the number of lines with field ai = 0 # count the number of lines with antenna ti = 0 # count the number of lines with timerange coi = 0 # count the number of lines with correlation ii = 0 # count the number of lines with intent fei = 0 # count the number of lines with feed ari = 0 # count the number of lines with array ui = 0 # count the number of lines with uvrange pi = 0 # count the number of lines with spw oi = 0 # count the number of lines with observation # for i in range(nrows): for k in cmddict.keys(): # cmdline = cmdlist[i] cmdline = cmddict[k]['command'] if cmdline.startswith('#'): ci += 1 npars['comment'] = ci continue # split by white space keyvlist = cmdline.split() if keyvlist.__len__() > 0: # Split by '=' for keyv in keyvlist: # Skip if it is a comment character # if keyv.count('#') > 0: break (xkey,xval) = keyv.split('=',1) # Remove quotes if type(xval) == str: if xval.count("'") > 0: xval = xval.strip("'") if xval.count('"') > 0: xval = xval.strip('"') # Remove blanks xval=xval.replace(' ',''); # Check which parameter, if not empty if xval != "": if xkey == "scan": si += 1 npars['scan'] = si elif xkey == "field": fi += 1 npars['field'] = fi elif xkey == "antenna": ai += 1 npars['antenna'] = ai elif xkey == "timerange": ti += 1 npars['timerange'] = ti elif xkey == "correlation": coi += 1 npars['correlation'] = coi elif xkey == "intent": ii += 1 npars['intent'] = ii elif xkey == "feed": fei += 1 npars['feed'] = fei elif xkey == "array": ari += 1 npars['array'] = ari elif xkey == "uvrange": ui += 1 npars['uvrange'] = ui elif xkey == "spw": pi += 1 npars['spw'] = pi elif xkey == "observation": oi += 1 npars['observation'] = oi return npars def compressSelectionList(vis='',dicpars={}): """ - Find a loose union of data-selection parameters, to reduce the MSSelection parsing load. - This compressed selection list is only meant to be used with af.selectdata(), because further selections are handled internally. - Use MSSelection in its 'onlyparse=True' mode to gather a list of fields, spws, antennas touched by the selection list. These are the only keys for which MSSelection does not need to parse the MS, and will cover the expensive cases where complicated antenna and spw expressions can slow MSSelection down. """ from numpy import unique; mslocal.open(vis) #, nomodify=False) try: indices = mslocal.msseltoindex(vis=vis,field=dicpars['field'], spw=dicpars['spw'],baseline=dicpars['antenna']); finally: mslocal.close() c_field = str(list(unique(indices['field']))).strip('[]'); c_spw = str(list(unique(indices['spw']))).strip('[]'); c_antenna = str(list( unique( list(indices['antenna1']) + list(indices['antenna2']) ) ) ).strip('[]'); dicpars['field'] = c_field; dicpars['spw'] = c_spw; dicpars['antenna'] = c_antenna; # Programmer note : Other selection parameters that can be compressed accurately # from MS subtable information alone (no need to parse the main table) are # 'array', 'observationid', 'state(or intent)'. They are currently un-available # via mslocal.msseltoindex() and therefore not used here yet. return; # ONLY for flagcmd task def writeFlagCmd(msfile, myflags, vrows, applied, add_reason, outfile): ''' Writes the flag commands to FLAG_CMD or to an ASCII file. Used only by the flagcmd task. This uses the old, simple parser. For flagdata, use writeFlagCommands(). msfile --> MS myflags --> dictionary of commands read from inputfile (from readFromTable, etc.) vrows --> list of valid rows from myflags dictionary to save applied --> value to update APPLIED column of FLAG_CMD add_reason --> reason to add to output (replace input reason, if any) outfile --> if not empty, save to it Returns the number of commands written to output ''' nadd = 0 try: import pylab as pl except ImportError as exc: raise ImportError('Failed to load pylab, required in writeFlagCmd: {}'. format(exc)) # append to a file if outfile != '': ffout = open(outfile, 'a') # Replace blanks from reason and cmdreason with underscores # and warn the user. if add_reason != '': tempreason = add_reason add_reason = tempreason.replace(' ','_') casalog.post('Replaced blanks with underscores in cmdreason', 'WARN') try: for key in myflags.keys(): # Remove leading and trailing white spaces cmdline = myflags[key]['command'].strip() # Add addantenna parameter back if myflags[key].__contains__('addantenna'): addantenna = myflags[key]['addantenna'] cmdline = cmdline + ' addantenna=' + str(addantenna) + ' ' reason = myflags[key]['reason'] if add_reason != '': reason = str(add_reason) if reason != '': cmdline = cmdline + ' reason=' + str(reason) + ' ' # For flagcmd saving newdict = evalString(cmdline) # newdict = evaluateString(cmdline) cmdline = '' for k,v in newdict.items(): cmdstr = "" # Add quotes to non-quoted strings if isinstance(v, str): if v.count("'") > 0: v = v.strip("'") if v.count('"') > 0: v = v.strip('"') # Add quotes to string values cmdstr = "'"+v+"'" v = cmdstr cmdline = cmdline + k + '=' + str(v) + ' ' elif isinstance(v,list): lstring = '' for ll in v: if isinstance(ll,list): for nl in ll: lstring = lstring + str(nl)+',' else: lstring = lstring + str(ll)+',' lstring = lstring.rstrip(',') cmdline = cmdline + k + '=['+ lstring + '] ' else: cmdline = cmdline + k + '=' + str(v) + ' ' # Save to output file ffout.write('%s\n' %cmdline.rstrip()) # except: except Exception as instance: raise Exception('Error writing lines to file %s. %s' %(outfile,instance)) ffout.close() return # Append new commands to existing table if vrows.__len__() > 0: # Extract flags from dictionary into list tim_list = [] intv_list = [] cmd_list = [] reas_list = [] typ_list = [] sev_list = [] lev_list = [] app_list = [] # Only write valid rows that have been applied to MS for key in vrows: # do not write line with summary mode command = myflags[key]['command'] if command.__contains__('summary'): continue # Add addantenna back if myflags[key].__contains__('addantenna'): addantenna = myflags[key]['addantenna'] command = command + ' addantenna=' + str(addantenna) # Evaluate the types newdict = evalString(command) # newdict = evaluateString(command) cmdline = '' for k,v in newdict.items(): cmdstr = "" if isinstance(v, str): if v.count("'") > 0: v = v.strip("'") if v.count('"') > 0: v = v.strip('"') # Add quotes to string values cmdstr = "'"+v+"'" v = cmdstr cmdline = cmdline + k + '=' + str(v) + ' ' elif isinstance(v,list): lstring = '' for ll in v: lstring = lstring + str(ll)+',' lstring = lstring.rstrip(',') cmdline = cmdline + k + '=['+ lstring + '] ' else: cmdline = cmdline + k + '=' + str(v) + ' ' cmd_list.append(cmdline) tim_list.append(myflags[key]['time']) intv_list.append(myflags[key]['interval']) if add_reason != '': reas_list.append(add_reason) else: reas_list.append(myflags[key]['reason']) typ_list.append(myflags[key]['type']) sev_list.append(myflags[key]['severity']) lev_list.append(myflags[key]['level']) app_list.append(applied) # Save to FLAG_CMD table nadd = cmd_list.__len__() mstable = msfile + '/FLAG_CMD' try: tblocal.open(mstable, nomodify=False) except: raise Exception('Error opening FLAG_CMD table ' + mstable) nrows = int(tblocal.nrows()) casalog.post('There are ' + str(nrows) + ' rows already in FLAG_CMD', 'DEBUG') # add blank rows tblocal.addrows(nadd) # now fill them in tblocal.putcol('TIME', numpy.array(tim_list), startrow=nrows, nrow=nadd) tblocal.putcol('INTERVAL', numpy.array(intv_list), startrow=nrows, nrow=nadd) tblocal.putcol('REASON', numpy.array(reas_list), startrow=nrows, nrow=nadd) tblocal.putcol('COMMAND', numpy.array(cmd_list), startrow=nrows, nrow=nadd) # Other columns tblocal.putcol('TYPE', numpy.array(typ_list), startrow=nrows, nrow=nadd) tblocal.putcol('SEVERITY', numpy.array(sev_list), startrow=nrows, nrow=nadd) tblocal.putcol('LEVEL', numpy.array(lev_list), startrow=nrows, nrow=nadd) tblocal.putcol('APPLIED', numpy.array(app_list), startrow=nrows, nrow=nadd) nrows = int(tblocal.nrows()) tblocal.close() casalog.post('Saved ' + str(nrows) + ' rows to FLAG_CMD') else: casalog.post('Saved zero rows to FLAG_CMD; no flags found') return nrows def getReason(cmdline): '''Get the reason values from a line with strings cmdline --> a string with parameters returns a string with reason values. ''' reason = '' # Skip comment lines if cmdline.startswith('#'): return reason # Split by white space keyvlist = cmdline.split() if keyvlist.__len__() > 0: for keyv in keyvlist: # Split by '=' (xkey,xval) = keyv.split('=',1) # Remove quotes if type(xval) == str: if xval.count("'") > 0: xval = xval.strip("'") if xval.count('"') > 0: xval = xval.strip('"') # Check if reason is in if xkey == "reason": reason = xval break; return reason def getLinePars(cmdline, mlist=[]): '''Get a dictionary of all selection parameters from a line: cmdline --> a string with parameters mlist --> a list of mode's parameters to add to the output dictionary Returns a dictionary. ''' # Dictionary of parameters to return dicpars = {} # Skip comment lines if cmdline.startswith('#'): return dicpars # split by white space keyvlist = cmdline.split() if keyvlist.__len__() > 0: # Split by '=' for keyv in keyvlist: (xkey,xval) = keyv.split('=',1) # Remove quotes if type(xval) == str: if xval.count("'") > 0: xval = xval.strip("'") if xval.count('"') > 0: xval = xval.strip('"') # Check which parameter if xkey == "scan": dicpars['scan'] = xval elif xkey == "field": dicpars['field'] = xval elif xkey == "antenna": dicpars['antenna'] = xval elif xkey == "timerange": dicpars['timerange'] = xval elif xkey == "correlation": dicpars['correlation'] = xval.upper() elif xkey == "intent": dicpars['intent'] = xval elif xkey == "feed": dicpars['feed'] = xval elif xkey == "array": dicpars['array'] = xval elif xkey == "uvrange": dicpars['uvrange'] = xval elif xkey == "spw": dicpars['spw'] = xval elif xkey == "observation": dicpars['observation'] = xval elif xkey == "mode": if xval == 'manualflag': xval = 'manual' dicpars['mode'] = xval elif mlist != []: # Any parameters requested for this mode? for m in mlist: if xkey == m: dicpars[m] = xval casalog.post(':getLinePars::dicpars=%s'%dicpars, 'DEBUG') return dicpars def getSelectionPars(cmdline): '''Get a dictionary of all selection parameters from a line: cmdline --> a string with parameters ''' # Dictionary of parameters to return dicpars = {} # Skip comment lines if cmdline.startswith('#'): return dicpars # split by white space keyvlist = cmdline.split() if keyvlist.__len__() > 0: # Split by '=' for keyv in keyvlist: (xkey,xval) = keyv.split('=',1) # Remove quotes if type(xval) == str: if xval.count("'") > 0: xval = xval.strip("'") if xval.count('"') > 0: xval = xval.strip('"') # Check which parameter if xkey == "scan": dicpars['scan'] = xval elif xkey == "field": dicpars['field'] = xval elif xkey == "antenna": dicpars['antenna'] = xval elif xkey == "timerange": dicpars['timerange'] = xval # Correlation will be handled by the agent elif xkey == "correlation": dicpars['correlation'] = '' elif xkey == "intent": dicpars['intent'] = xval elif xkey == "feed": dicpars['feed'] = xval elif xkey == "array": dicpars['array'] = xval elif xkey == "uvrange": dicpars['uvrange'] = xval elif xkey == "spw": dicpars['spw'] = xval elif xkey == "observation": dicpars['observation'] = xval return dicpars def readNtime(params): '''Check the value and units of ntime params --> dictionary of agent's parameters ''' newtime = 0.0 if 'ntime' in params: ntime = params['ntime'] # Verify the ntime value if type(ntime) == float or type(ntime) == int: if ntime <= 0: raise Exception('Parameter ntime cannot be < = 0') else: # units are seconds newtime = float(ntime) elif type(ntime) == str: if ntime == 'scan': # iteration time step is a scan newtime = 0.0 else: # read the units from the string qtime = qalocal.quantity(ntime) if qtime['unit'] == 'min': # convert to seconds qtime = qalocal.convert(qtime, 's') elif qtime['unit'] == '': qtime['unit'] = 's' # check units if qtime['unit'] == 's': newtime = qtime['value'] else: casalog.post('Cannot convert units of ntime. Will use default 0.0s', 'WARN') params['ntime'] = float(newtime) def fixType(params): '''Give correct types to non-string parameters The types are defined in the XML file of the task flagdata Do not repeat any parameter''' # manual parameter if 'autocorr' in params: params['autocorr'] = eval(params['autocorr'].capitalize()) # quack parameters if 'quackmode' in params and not params['quackmode'] in ['beg' , 'endb', 'end', 'tail']: raise Exception( \ "Illegal value '%s' of parameter quackmode, must be either 'beg', 'endb', 'end' or 'tail'" \ % params['quackmode']) if 'quackinterval' in params: params['quackinterval'] = float(params['quackinterval']) if 'quackincrement' in params: if type(params['quackincrement']) == str: params['quackincrement'] = eval(params['quackincrement'].capitalize()) # clip parameters if 'clipminmax' in params: value01 = params['clipminmax'] # turn string into [min,max] range value0 = value01.lstrip('[') value = value0.rstrip(']') r = value.split(',') rmin = float(r[0]) rmax = float(r[1]) params['clipminmax'] = [rmin, rmax] if 'clipoutside' in params: if type(params['clipoutside']) == str: params['clipoutside'] = eval(params['clipoutside'].capitalize()) else: params['clipoutside'] = params['clipoutside'] if 'channelavg' in params: params['channelavg'] = eval(params['channelavg'].capitalize()) if 'clipzeros' in params: params['clipzeros'] = eval(params['clipzeros'].capitalize()) # shadow parameter if 'tolerance' in params: params['tolerance'] = float(params['tolerance']) # elevation parameters if 'lowerlimit' in params: params['lowerlimit'] = float(params['lowerlimit']) if 'upperlimit'in params: params['upperlimit'] = float(params['upperlimit']) # extend parameters if 'extendpols' in params: params['extendpols'] = eval(params['extendpols'].capitalize()) if 'growtime' in params: params['growtime'] = float(params['growtime']) if 'growfreq' in params: params['growfreq'] = float(params['growfreq']) if 'growaround' in params: params['growaround'] = eval(params['growaround'].capitalize()) if 'flagneartime' in params: params['flagneartime'] = eval(params['flagneartime'].capitalize()) if 'flagnearfreq' in params: params['flagnearfreq'] = eval(params['flagnearfreq'].capitalize()) # tfcrop parameters if 'combinescans' in params: params['combinescans'] = eval(params['combinescans'].capitalize()) if 'timecutoff' in params: params['timecutoff'] = float(params['timecutoff']) if 'freqcutoff' in params: params['freqcutoff'] = float(params['freqcutoff']) if 'maxnpieces' in params: params['maxnpieces'] = int(params['maxnpieces']) if 'halfwin' in params: params['halfwin'] = int(params['halfwin']) if 'extendflags' in params: params['extendflags'] = eval(params['extendflags'].capitalize()) # rflag parameters if 'winsize' in params: params['winsize'] = int(params['winsize']); if 'timedev' in params: timepar = params['timedev'] try: timepar = eval(timepar) except Exception: timepar = readRFlagThresholdFile(params['timedev'],'timedev'); params['timedev'] = timepar if 'freqdev' in params: freqpar = params['freqdev'] try: freqpar = eval(freqpar) except Exception: freqpar = readRFlagThresholdFile(params['freqdev'],'freqdev'); params['freqdev'] = freqpar if 'timedevscale' in params: params['timedevscale'] = float(params['timedevscale']); if 'freqdevscale' in params: params['freqdevscale'] = float(params['freqdevscale']); if 'spectralmin' in params: params['spectralmin'] = float(params['spectralmin']); if 'spectralmax' in params: params['spectralmax'] = float(params['spectralmax']); def purgeEmptyPars(cmdline): '''Remove empty parameters from a string: cmdline --> a string with parameters returns a string containing only parameters with values ''' newstr = '' # split by white space keyvlist = cmdline.split() if keyvlist.__len__() > 0: # Split by '=' for keyv in keyvlist: (xkey,xval) = keyv.split('=',1) # Remove quotes if type(xval) == str: if xval.count("'") > 0: xval = xval.strip("'") if xval.count('"') > 0: xval = xval.strip('"') # Write only parameters with values if xval == '': continue else: newstr = newstr+xkey+'='+xval+' ' else: casalog.post('String of parameters is empty','WARN') return newstr def purgeParameter(cmdline, par): '''Remove parameter from a string: cmdline --> a string with a parameter to be removed par --> the parameter to be removed from the string returns a string containing the remaining parameters ''' newstr = '' # split by white space keyvlist = cmdline.split() if keyvlist.__len__() > 0: # Split by '=' for keyv in keyvlist: (xkey,xval) = keyv.split('=',1) # Remove quotes if type(xval) == str: if xval.count("'") > 0: xval = xval.strip("'") if xval.count('"') > 0: xval = xval.strip('"') # Write only parameters with values if xkey == par: continue else: newstr = newstr+xkey+'=' + xval+ ' ' else: casalog.post('String of parameters is empty','WARN') return newstr def setupAgent(aflocal, myflagcmd, myrows, apply, writeflags, display=''): ''' Setup the parameters of each agent and call the agentflagger tool myflagcmd --> it is a dictionary coming from readFromTable, readFile, etc. myrows --> selected rows to apply/unapply flags apply --> it's a boolean to control whether to apply or unapply the flags writeflags --> used by mode=rflag only display --> used by mode='rflag only''' if not myflagcmd.__len__() >0: casalog.post('There are no flag cmds in list', 'SEVERE') return # Parameters for each mode manualpars = ['autocorr'] unflagpars = [] clippars = ['clipminmax', 'clipoutside','datacolumn', 'channelavg', 'clipzeros'] quackpars = ['quackinterval','quackmode','quackincrement'] shadowpars = ['tolerance', 'addantenna'] elevationpars = ['lowerlimit','upperlimit'] tfcroppars = ['ntime','combinescans','datacolumn','timecutoff','freqcutoff', 'timefit','freqfit','maxnpieces','flagdimension','usewindowstats','halfwin', 'extendflags'] antintpars = ['antint_ref_antenna','minchanfrac','verbose'] extendpars = ['ntime','combinescans','extendpols','growtime','growfreq','growaround', 'flagneartime','flagnearfreq'] rflagpars = ['winsize','timedev','freqdev','timedevscale','freqdevscale','spectralmax', 'spectralmin', 'extendflags'] # dictionary of successful command lines to save to outfile savelist = {} # Setup the agent for each input line for key in myflagcmd.keys(): cmdline = myflagcmd[key]['command'] applied = myflagcmd[key]['applied'] interval = myflagcmd[key]['interval'] level = myflagcmd[key]['level'] reason = myflagcmd[key]['reason'] severity = myflagcmd[key]['severity'] coltime = myflagcmd[key]['time'] coltype = myflagcmd[key]['type'] if debug: casalog.post('cmdline for key%s'%key) casalog.post('%s'%cmdline) casalog.post('applied is %s'%applied) if cmdline.startswith('#'): continue modepars = {} parslist = {} mode = '' valid = True addantenna = {} # Get the specific parameters for the mode if cmdline.__contains__('mode'): if cmdline.__contains__('manual'): mode = 'manual' modepars = getLinePars(cmdline,manualpars) elif cmdline.__contains__('clip'): mode = 'clip' modepars = getLinePars(cmdline,clippars) elif cmdline.__contains__('quack'): mode = 'quack' modepars = getLinePars(cmdline,quackpars) elif cmdline.__contains__('shadow'): mode = 'shadow' modepars = getLinePars(cmdline,shadowpars) # Get addantenna dictionary if myflagcmd[key].__contains__('addantenna'): addantenna = myflagcmd[key]['addantenna'] modepars['addantenna'] = addantenna else: # Get antenna filename if (modepars.__contains__('addantenna')): ant_par = modepars['addantenna'] # It must be a string if (type(ant_par) == str and ant_par != ''): antennafile = modepars['addantenna'] addantenna = readAntennaList(antennafile) modepars['addantenna'] = addantenna elif cmdline.__contains__('elevation'): mode = 'elevation' modepars = getLinePars(cmdline,elevationpars) elif cmdline.__contains__('tfcrop'): mode = 'tfcrop' modepars = getLinePars(cmdline,tfcroppars) elif cmdline.__contains__('antint'): mode = 'antint' modepars = getLinePars(cmdline,antintpars) elif cmdline.__contains__('extend') and cmdline.__contains__('extendpols'): # Necessary to avoid matching the extenflags parameter of rflag/tfcrop mode = 'extend' modepars = getLinePars(cmdline,extendpars) elif cmdline.__contains__('unflag'): mode = 'unflag' modepars = getLinePars(cmdline,unflagpars) elif cmdline.__contains__('rflag'): mode = 'rflag' modepars = getLinePars(cmdline,rflagpars) ##### ### According to 'shadow' file handling, this code should be here... ### but, it's already done inside fixType. ##### #if( type(modepars['timedev']) == str and writeflags == True): # timedev = readRFlagThresholdFile(modepars['timedev'],'timedev') # modepars['timedev'] = timedev #if( type(modepars['freqdev']) == str and writeflags == True): # freqdev = readRFlagThresholdFile(modepars['freqdev'],'freqdev') # modepars['freqdev'] = freqdev # Add the writeflags and display parameters modepars['writeflags'] = writeflags modepars['display'] = display else: # Unknown mode, ignore it casalog.post('Ignoring unknown mode', 'WARN') # valid = False else: # No mode means manual mode = 'manual' cmdline = cmdline+' mode=manual' modepars = getLinePars(cmdline,manualpars) # Read ntime readNtime(modepars) # Cast the correct type to non-string parameters fixType(modepars) # Add the apply/unapply parameter to dictionary modepars['apply'] = apply # Unapply selected rows only and re-apply the other rows with APPLIED=True # if not apply and myrows.__len__() > 0: # if key in myrows: # modepars['apply'] = False # elif not applied: # casalog.post("Skipping this %s"%modepars,"DEBUG") # continue # elif applied: # modepars['apply'] = True # valid = False # Keep only cmds that overlap with the unapply cmds # TODO later # Hold the name of the agent and the cmd row number agent_name = mode.capitalize()+'_'+str(key) modepars['name'] = agent_name # Remove the data selection parameters if there is only one agent for performance reasons. # Explanation: if only one agent exists and the data selection parameters are parsed to it, # it will have to go through the entire MS and check if each data selection given to the agent # matches what the user asked in the selected data. # Only correlation, antenna and timerange will go to the agent # CAS-3959 Handle channel selection at the FlagAgent level, leave spw in here too if myflagcmd.__len__() == 1: sellist=['scan','field','intent','feed','array','uvrange','observation'] for k in sellist: if k in modepars: modepars.pop(k) casalog.post('Parsing parameters of mode %s in row %s'%(mode,key), 'DEBUG') casalog.post('%s'%modepars, 'DEBUG') if debug: casalog.post('Parsing parameters of mode %s in row %s'%(mode,key)) casalog.post(modepars) # Parse the dictionary of parameters to the tool if (not aflocal.parseagentparameters(modepars)): casalog.post('Failed to parse parameters of mode %s in row %s' %(mode,key), 'WARN') continue # Save the dictionary of valid agents if valid: # add this command line to list to save in outfile parslist['row'] = key parslist['command'] = cmdline parslist['applied'] = applied parslist['interval'] = interval parslist['level'] = level parslist['reason'] = reason parslist['severity'] = severity parslist['time'] = coltime parslist['type'] = coltype if addantenna != {}: parslist['addantenna'] = addantenna savelist[key] = parslist if debug: casalog.post('Dictionary of valid commands to save') casalog.post('%s'%savelist) return savelist def backupFlags(aflocal=None, msfile='', prename='flagbackup'): '''Create a backup of the FLAG column aflocal local version of the agentflagger tool or msfile name of MS/cal table to backup prename prefix for name of flag backup file If msfile is given, aflocal will not be used Create names like this: flags.flagcmd_1, flags.flagdata_1, Generally <task>_<i>, where i is the smallest integer giving a name, which does not already exist''' if msfile != '': # open msfile and attach it to tool aflocal = agentflagger() aflocal.open(msfile) elif aflocal == None: casalog.post('Need an MS or a copy of the agentflagger tool to create a backup','WARN') return prefix = prename try: existing = aflocal.getflagversionlist(printflags=False) # remove comments from strings existing = [x[0:x.find(' : ')] for x in existing] i = 1 while True: versionname = prefix + '_' + str(i) if not versionname in existing: break else: i = i + 1 time_string = str(time.strftime('%Y-%m-%d %H:%M:%S')) casalog.post('Saving current flags to ' + versionname, 'DEBUG') aflocal.saveflagversion(versionname=versionname, comment='Flags autosave on ' + time_string, merge='replace') finally: if msfile != '': aflocal.done() return #### #### Set of functions to handle antenna information for shadowing. #### #### - extractAntennaInfo : Extract info into a returned dictionary (and text file) #### - antListWrite : Write the dictionary to a text file #### - antListRead : Read the text file and return a dictionary #### #### Example : #### alist = extractAntennaInfo(msname='../Data/shadowtest.ms', #### antnamelist=['VLA1','VLA15'], outfile='ttt.txt') #### #### alist = antListRead('ttt.txt'); #### ################################### #### Example output text file ( 'ttt.txt' ) ################################### # #name = VLA1 #diameter = 25.0 #position = [-1601144.961466915, -5041998.0197185818, 3554864.76811967] #name = VLA15 #diameter = 25.0 #position = [-1601556.245351332, -5041990.7252590274, 3554684.6464035073] # ################################### #### Example output dictionary ( alist ) ################################### # #{'0': {'diameter ': 25.0, # 'name ': 'VLA1', # 'position ': [-1601144.961466915, # -5041998.0197185818, # 3554864.7681196001]}, # '1': {'diameter ': 25.0, # 'name ': 'VLA15', # 'position ': [-1601556.245351332, # -5041990.7252590274, # 3554684.6464035069]}} # ################################## def extractAntennaInfo(msname='', antnamelist=[], outfile=''): """ Function to extract antenna names, positions, and diameters for a specified subset of the ANTENNA subtable of the specified MS. - It writes a text file, which can be sent as input for shadow in the task - It also returns a dictionary, which can be sent as input for shadow in the tool. This dictionary can also be given as input in the task. msname : name of MS antennalist : list of strings (antenna names). Names must match exactly. Case insensitive. outfile : name of text file. Will be overwritten if exists. If outfile='', no output file is written Always returns a dictionary containing the same info as in the file Example : antinfo = extractAntennaInfo(msname='xxx.ms',antnamelist=['vla1','vla2'],outfile='out.txt'); """ ## Check that the MS exists if(not os.path.exists(msname)): casalog.post("Cannot find MS : {}".format(msname)) return False; ## If outfile exists, delete it if(os.path.exists(outfile)): casalog.post("Replacing existing file : {}".format(outfile)) rmcmd = "rm -rf "+outfile; os.system(rmcmd); ## Convert input antenna names to upper-case newants=[]; for ants in antnamelist: newants.append( ants.upper() ); antnamelist = newants; ## Read antenna subtable of input MS tblocal.open(msname+'/ANTENNA'); a_position = (tblocal.getcol('POSITION')).transpose(); a_dish_diameter = tblocal.getcol('DISH_DIAMETER'); a_name = tblocal.getcol('NAME'); tblocal.close(); ## Pick out only selected antennas from this list, and make a dictionary antlist = {}; counter=0; for antid in range(0, len(a_name)): if (a_name[antid]).upper() in antnamelist: antlist[str(counter)] = { 'name':a_name[antid] , 'position': list(a_position[antid]) , 'diameter': a_dish_diameter[antid] } ; counter=counter+1; ## Open a new file and write this info into it, if requested if(outfile != ''): casalog.post("Making new file : {}".format(outfile)) writeAntennaList(outfile, antlist); ## always return the dictionary anyway. return antlist; ############################################## def writeAntennaList(outfile='', antlist={}): """ Save the antlist dictionary as a text file """ ofile = open(outfile, 'w'); for apid in sorted(antlist): apars = antlist[apid]; ofile.write("name=" + str(apars['name']) + '\n'); ofile.write("diameter=" + str(apars['diameter'])+'\n'); ofile.write("position=" + str((apars['position']))+'\n'); ofile.close(); ############################################## def readAntennaList(infile=''): """ Read the antlist text file and return a dictionary A return value of empty {} indicates an error (or, empty file). The file needs to have 3 entries per antenna, on separate lines. The diameter and position are in units of meters, with positions in ITRF. Multiple antennas can be specified by repeating these three lines. Blank lines are allowed. Lines can be commented with '#' as the first character. Example : name = ea05 diameter = 25.0 position = [-1601144.96146691, -5041998.01971858, 3554864.76811967] """ if (type(infile) == str) & os.path.exists(infile): try: ifile = open(infile,'r'); except: raise Exception('Error opening file ' + infile) thelist = ifile.readlines(); ifile.close(); else: raise Exception( \ 'File %s not found - please verify the name'%infile) cleanlist=[]; for aline in thelist: if(len(aline)>5 and aline[0] != '#'): cleanlist.append(aline.rstrip()); #casalog.post('Found ' + str(len(cleanlist)) + ' valid lines out of ' + str(len(thelist));) if( len(cleanlist) > 0 and len(cleanlist) % 3 != 0 ): casalog.post("\nThe file needs to have 3 entries per antenna, on separate lines. For example :") casalog.post("name=ea05") casalog.post("diameter=25.0") casalog.post("position=[-1601144.96146691, -5041998.01971858, 3554864.76811967]") casalog.post("\n") casalog.post("The diameter and position are in units of meters, with positions in ITRF") return False; antlist={}; counter=0; for aline in range(0,len(cleanlist),3): antdict = {}; for row in range(0,3): pars = cleanlist[aline+row].split("=",1); #casalog.post(aline, row, pars) if(len(pars) != 2): casalog.post('Error in parsing : ' + cleanlist[aline+row], 'ERROR') return {}; else: if(pars[0].count('name') > 0 ): antdict[pars[0].rstrip()] = str(pars[1].rsplit()[0]); if(pars[0].count('diameter') > 0 ): antdict[pars[0].rstrip()] = float(pars[1]); if(pars[0].count('position') > 0 ): plist = pars[1][1:-2].replace('[','').split(','); if(len(plist) != 3): casalog.post('Error in parsing : ' + cleanlist[aline+row], 'ERROR') return {}; else: qlist=[]; for ind in range(0,3): qlist.append(float(plist[ind])); antdict[pars[0].rstrip()] = qlist; antlist[str(counter)] = antdict; counter = counter+1; return antlist; ################################################ # # Function to pull out RFLAG thresholds from the returned report dictionary. # def writeRFlagThresholdFile(rflag_thresholds={},timedevfile='', freqdevfile='',agent_id=0): """ Extract the RFLAG output thresholds from the threshold dictionary Return them as arrays, and optionally, write them into a file. """ # Decide the output file name. if( type(timedevfile) == str and timedevfile != '' and timedevfile.count('[')==0 and (not timedevfile.replace('.','').isdigit() ) ): toutfile = timedevfile else: toutfile = 'rflag_output_thresholds_timedev'+str(agent_id)+'.txt' # Decide the output file name. if( type(freqdevfile) == str and freqdevfile != '' and freqdevfile.count('[')==0 and (not freqdevfile.replace('.','').isdigit() ) ): foutfile = freqdevfile else: foutfile = 'rflag_output_thresholds_freqdev'+str(agent_id)+'.txt' # save rflag output in file, and print them everywhere. casalog.post("Saving RFlag_"+str(agent_id)+" output in : " + toutfile + " and " + foutfile, 'INFO') ofiletime = open(toutfile, 'w'); ofilefreq = open(foutfile, 'w'); # Construct dictionary from what needs to be stored. timedict = {'name':rflag_thresholds['name'] , 'timedev': (rflag_thresholds['timedev']).tolist()} freqdict = {'name':rflag_thresholds['name'] , 'freqdev': (rflag_thresholds['freqdev']).tolist()} timestr = convertDictToString(timedict) freqstr = convertDictToString(freqdict) # Write to file ofiletime.write(timestr + '\n'); ofilefreq.write(freqstr + '\n'); # Send to logger casalog.post("RFlag_"+str(agent_id)+" output timedev written to " + toutfile + " : " + timestr, 'INFO'); casalog.post("RFlag_"+str(agent_id)+" output freqdev written to " + foutfile + " : " + freqstr, 'INFO'); # End filed ofiletime.write('\n'); ofiletime.close(); ofilefreq.write('\n'); ofilefreq.close(); # Returne timedev, freqdev contents. This is for savepars later. return timedict['timedev'], freqdict['freqdev'] ############################################## def readRFlagThresholdFile(infile='',inkey=''): """ Read the input RFlag threshold file, and return dictionaries. """ if(infile==''): return []; if ( not os.path.exists(infile) ): casalog.post('Cannot find file : ' + infile) return []; ifile = open(infile,'r'); thelist = ifile.readlines(); ifile.close(); cleanlist=[]; for aline in thelist: if(len(aline)>5 and aline[0] != '#'): cleanlist.append(aline.rstrip().rstrip('\n')) # threshlist={}; threshlist = OrderedDict() for aline in range(0,len(cleanlist)): threshlist[str(aline)] = convertStringToDict(cleanlist[aline]); if inkey in threshlist[str(aline)]: devthreshold = threshlist[str(aline)][inkey] # return only the last one. There should be only one anyway. return devthreshold ############################################## ## Note - replace all arrays by lists before coming here. def convertDictToString(indict={}): '''Convert dictionary to string''' thestr = str(indict); # Remove newlines and spaces from this string. thestr = thestr.replace('\n',''); thestr = thestr.replace(' ',''); return thestr; ############################################## def convertStringToDict(instr=''): '''Convert string to dictionary''' instr = instr.replace('\n',''); try: thedict = OrderedDict() thedict = ast.literal_eval(instr) except Exception as instance: casalog.post("*** Error converting string %s to dictionary : \'%s\'" % (instr,instance),'ERROR'); return thedict; ############################################## def extractRFlagOutputFromSummary(mode,summary_stats_list, flagcmd): """ Function to pull out 'rflag' output from the long dictionary, and (1) write the output files with thresholds. If the filename is specified, use it. If filename is not specified, make one up. (2) modify entries in 'cmdline' so that it is ready for savepars. This is to ensure that 'savepars' saves the contents of the threshold-files and not just the file-names. It has to save it in the form that flagdata accepts inline : e.g. timedev=[[1,10,0.1],[1,11,0.07]] . This way, the user need not keep track of threshold text files if they use 'savepars' with action='apply'. """ if type(summary_stats_list) is dict: nreps = summary_stats_list['nreport'] for rep in range(0,nreps): repname = 'report'+str(rep) if summary_stats_list[repname]['type'] == "rflag": # Pull out the rflag threshold dictionary. This has a 'name' in it. rflag_thresholds = summary_stats_list[repname] ##casalog.post(rflag_thresholds) # Get the rflag id, to later construct a 'name' from to match the above. rflagid = 0 if mode=='list': rflagid = int( rflag_thresholds['name'].replace('Rflag_','') ) # Go through the flagcmd list, to find the 'rflags'..... for key in flagcmd.keys(): cmdline = flagcmd[key]['command']; if cmdline.__contains__('rflag'): # Check for match between input flagcmd and output threshold, via the rflag id if(key==rflagid): # If timedev,freqdev are missing from cmdline, add empty ones. if( not cmdline.__contains__('timedev=') ): # aah. don't confuse it with timedevscale cmdline = cmdline + " timedev=[] "; if( not cmdline.__contains__('freqdev=') ): cmdline = cmdline + " freqdev=[] "; # Pull out timedev, freqdev strings from flagcmd rflagpars = getLinePars(cmdline , ['timedev','freqdev','writeflags']); ##casalog.post("cmdline : "+cmdline) # Write RFlag thresholds to these file names. newtimedev,newfreqdev = writeRFlagThresholdFile(rflag_thresholds, rflagpars['timedev'], rflagpars['freqdev'], rflagid) ## Modify the flagcmd string, so that savepars sees the contents of the file if( rflagpars['timedev'].__contains__('[') ): oldstring = 'timedev='+str(rflagpars['timedev']) newstring = 'timedev='+str(newtimedev).replace(' ','') ##casalog.post( "time : replacing " + oldstring + newstring) cmdline = cmdline.replace( oldstring, newstring ); if( rflagpars['freqdev'].__contains__('[') ): oldstring = 'freqdev='+str(rflagpars['freqdev']) newstring = 'freqdev='+str(newfreqdev).replace(' ','') ##casalog.post("freq : replacing " + oldstring + newstring) cmdline = cmdline.replace( oldstring, newstring ); # Remove writeflags from the cmd to prevent it from going into savepars oldstring = 'writeflags='+str(rflagpars['writeflags']) cmdline = cmdline.replace( oldstring, "" ); flagcmd[key]['command'] = cmdline; # Only used in writeFlagCmd def evalString(cmdline): '''Evaluate the correct types in a string with parameters''' # newcmdline = "" cmddict = OrderedDict() try: pairs = cmdline.split() except: raise "Whitespace in parameter values is not yet supported" for kv in pairs: newval = None (key,val) = kv.split('=',1) # Selection parameters are always string if key == 'spw' or key == 'scan' or key == 'field' or \ key == 'antenna' or key == 'correlation' or key == 'array' or \ key == 'timerange' or key == 'uvrange' or key == 'feed': newval = str(val) cmddict[key] = newval continue if val.startswith('['): newval = ast.literal_eval(val) elif val.startswith('{'): newval = eval(val) elif val == 'True' or val == 'False': if eval(val) == True or eval(val) == False: newval = bool(val) if newval == None: try: int(val) newval = int(val) except: # string newval = str(val) if newval == None: try: float(val) newval = float(val) except: # string newval = str(val) cmddict[key] = newval return cmddict