///# AgentFlagger.cc: this defines AgentFlagger //# Copyright (C) 2000,2001,2002 //# Associated Universities, Inc. Washington DC, USA. //# //# This library is free software; you can redistribute it and/or modify it //# under the terms of the GNU Library General Public License as published by //# the Free Software Foundation; either version 2 of the License, or (at your //# option) any later version. //# //# This library is distributed in the hope that it will be useful, but WITHOUT //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public //# License for more details. //# //# You should have received a copy of the GNU Library General Public License //# along with this library; if not, write to the Free Software Foundation, //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA. //# //# Correspondence concerning AIPS++ should be addressed as follows: //# Internet email: aips2-request@nrao.edu. //# Postal address: AIPS++ Project Office //# National Radio Astronomy Observatory //# 520 Edgemont Road //# Charlottesville, VA 22903-2475 USA //# //# $Id$ #include <casa/Arrays/Vector.h> #include <casa/Arrays/ArrayMath.h> #include <casa/Arrays/ArrayLogical.h> #include <casa/BasicSL/Complex.h> #include <casa/Utilities/Regex.h> #include <casa/OS/HostInfo.h> #include <casa/Exceptions/Error.h> #include <flagging/Flagging/AgentFlagger.h> #include <flagging/Flagging/FlagVersion.h> #include <casa/stdio.h> #include <casa/math.h> #include <stdarg.h> #include <sstream> #include <iostream> #include <vector> using namespace casacore; namespace casa { const bool AgentFlagger::dbg = false; // ----------------------------------------------------------------------- // Default Constructor // ----------------------------------------------------------------------- AgentFlagger::AgentFlagger () { fdh_p = NULL; summaryAgent_p = NULL; #if ! defined(WITHOUT_DBUS) displayAgent_p = NULL; #endif done(); } // ----------------------------------------------------------------------- // Default Destructor // ----------------------------------------------------------------------- AgentFlagger::~AgentFlagger () { done(); } void AgentFlagger::done() { if(fdh_p){ delete fdh_p; fdh_p = NULL; } // Default values of parameters msname_p = ""; iterationApproach_p = FlagDataHandler::SUB_INTEGRATION; timeInterval_p = 0.0; isMS_p = true; combinescans_p = false; extendflags_p = true; spw_p = ""; scan_p = ""; field_p = ""; antenna_p = ""; timerange_p = ""; correlation_p = ""; intent_p = ""; feed_p = ""; array_p = ""; uvrange_p = ""; observation_p = ""; max_p = 0.0; timeset_p = false; iterset_p = false; timeAvg_p = false; dataselection_p = Record(); /* if (! dataselection_p.empty()) { Record temp; dataselection_p = temp; } */ if (! agentParams_p.empty()) { Record temp; agentParams_p = temp; } if(summaryAgent_p){ summaryAgent_p = NULL; } #if ! defined(WITHOUT_DBUS) if(displayAgent_p){ displayAgent_p = NULL; } #endif mode_p = ""; agents_config_list_p.clear(); agents_config_list_copy_p.clear(); agents_list_p.clear(); return; } // --------------------------------------------------------------------- // AgentFlagger::open // Create a FlagDataHandler object based on the input type: // MS or a calibration file. Open the MS or cal table and // attach it to the tool. // --------------------------------------------------------------------- bool AgentFlagger::open(String msname, Double ntime) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); if (msname.empty()) { os << LogIO::SEVERE << "No Measurement Set has been parsed" << LogIO::POST; return false; } msname_p = msname; if (ntime) timeInterval_p = ntime; max_p = timeInterval_p; if(dbg){ os << LogIO::NORMAL << "msname = " << msname_p << " ntime = " << timeInterval_p << LogIO::POST; } if(fdh_p) delete fdh_p; // create a FlagDataHandler object Table table(msname_p,TableLock(TableLock::AutoNoReadLocking)); TableInfo& info = table.tableInfo(); String type = info.type(); table.flush(); table.relinquishAutoLocks(true); table.unlock(); os << LogIO::NORMAL << "Table type is " << type << LogIO::POST; // For a measurement set if (type == "Measurement Set") { isMS_p = true; fdh_p = new FlagMSHandler(msname_p, iterationApproach_p, timeInterval_p); } // For a calibration file else { isMS_p = false; fdh_p = new FlagCalTableHandler(msname_p, iterationApproach_p, timeInterval_p); } // Open the MS fdh_p->open(); return true; } // --------------------------------------------------------------------- // AgentFlagger::selectData // Get a record with data selection parameters and // Parse it to the FlagDataHandler to select the data // --------------------------------------------------------------------- bool AgentFlagger::selectData(Record selrec) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); if (dbg) os << LogIO::NORMAL << "Called from selectData(Record)" << LogIO::POST; if (! fdh_p){ os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST; return false; } if (! selrec.empty()) { dataselection_p = selrec; // Check if all the parameters are in the record. If not, // use the default values if (dataselection_p.isDefined("spw")) dataselection_p.get("spw", spw_p); if (dataselection_p.isDefined("scan")) dataselection_p.get("scan", scan_p); if (dataselection_p.isDefined("field")) dataselection_p.get("field", field_p); if (dataselection_p.isDefined("antenna")) dataselection_p.get("antenna", antenna_p); if (dataselection_p.isDefined("timerange")) dataselection_p.get("timerange", timerange_p); if (dataselection_p.isDefined("correlation")) dataselection_p.get("correlation", correlation_p); if (dataselection_p.isDefined("intent")) dataselection_p.get("intent", intent_p); if (dataselection_p.isDefined("feed")) dataselection_p.get("feed", feed_p); if (dataselection_p.isDefined("array")) dataselection_p.get("array", array_p); if (dataselection_p.isDefined("uvrange")) dataselection_p.get("uvrange", uvrange_p); if (dataselection_p.isDefined("observation")) dataselection_p.get("observation", observation_p); } bool ret_status = true; // Set the data selection ret_status = fdh_p->setDataSelection(dataselection_p); if (!ret_status) { os << LogIO::SEVERE << "Failed to set the data selection." << LogIO::POST; return false; } // Select the data ret_status = fdh_p->selectData(); if (!ret_status) { os << LogIO::SEVERE << "Failed to select the data." << LogIO::POST; return false; } return true; } // --------------------------------------------------------------------- // AgentFlagger::selectData // Create a record with data selection parameters. // --------------------------------------------------------------------- bool AgentFlagger::selectData(String field, String spw, String array, String feed, String scan, String antenna, String uvrange, String timerange, String correlation, String intent, String observation) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); if (dbg) os << LogIO::NORMAL << "Called from selectData(String....)" << LogIO::POST; // Create a record with the parameters Record selection = Record(); selection.define("spw", spw); selection.define("scan", scan); selection.define("field", field); selection.define("antenna", antenna); selection.define("timerange", timerange); selection.define("correlation", correlation); selection.define("intent", intent); selection.define("feed", feed); selection.define("array", array); selection.define("uvrange", uvrange); selection.define("observation", observation); // Call the main selectData() method selectData(selection); return true; } // --------------------------------------------------------------------- // AgentFlagger::parseAgentParameters // Create a vector of agents and parameters // Each input record contains data selection parameters // and agent's specific parameters // --------------------------------------------------------------------- bool AgentFlagger::parseAgentParameters(Record agent_params) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); if (! fdh_p){ os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST; return false; } // Default values for some parameters String mode = ""; String agent_name = ""; Bool apply = true; String datacolumn = "DATA"; // create a temporary vector of agents std::vector<Record> listOfAgents; if(agent_params.empty()){ os << LogIO::SEVERE << "No agent's record has been provided." << LogIO::POST; return false; } // Use the given Record of parameters agentParams_p = agent_params; if (! agentParams_p.isDefined("mode")) { os << LogIO::WARN << "No mode has been provided" << LogIO::POST; return false; } // Get mode agentParams_p.get("mode", mode); // Validate mode against known modes if (! isModeValid(mode)){ os << LogIO::WARN << "Mode "<< mode << " is not valid or doesn't exist" << LogIO::POST; return false; } // Name for the logging output if (! agentParams_p.isDefined("agentname")){ agent_name = mode; agent_name.capitalize(); agentParams_p.define("agentname", agent_name); } agentParams_p.get("agentname", agent_name); os << LogIO::DEBUG1 << "Parsing parameters for agent: "<<mode<< LogIO::POST; // Validate datacolumn if (mode.compare("tfcrop") == 0 or mode.compare("clip") == 0 or mode.compare("rflag") == 0 or mode.compare("antint") == 0 or mode.compare("display") == 0){ if (agentParams_p.isDefined("datacolumn")) agentParams_p.get("datacolumn", datacolumn); os << LogIO::NORMAL << "Validating data column "<<datacolumn<<" based on input type"<< LogIO::POST; if(!validateDataColumn(datacolumn)){ os << LogIO::WARN << "Data column "<< datacolumn << " does not exist in input data" << LogIO::POST; return false; } os << LogIO::DEBUG1 << "Will use data column "<< datacolumn << LogIO::POST; agentParams_p.define("datacolumn", datacolumn); } // Enforce a default value for the apply parameter if (! agentParams_p.isDefined("apply")){ agentParams_p.define("apply", apply); } // If there is a tfcrop, extend or rflag agent in the list, // get the maximum value of ntime and the combinescans parameter if (mode.compare("tfcrop") == 0 or mode.compare("extend") == 0 or mode.compare("rflag") == 0) { Double ntime; if (agentParams_p.isDefined("ntime")){ agentParams_p.get("ntime", ntime); getMax(ntime); } // Get the combinescans parameter. If any of them is true, // it will be true for the whole list Bool combine = false; if (agentParams_p.isDefined("combinescans")) agentParams_p.get("combinescans", combine); combinescans_p = combinescans_p || combine; os << LogIO::DEBUG2 << "max ntime="<<max_p<<" and combinescans="<< combinescans_p << LogIO::POST; } // Activate async i/o if tfcrop/clip/rflag is present if (mode.compare("tfcrop") == 0 or mode.compare("clip") == 0 or mode.compare("rflag") == 0) { fdh_p->enableAsyncIO(true); } // Default for all modes String correlation = ""; // Make correlation always uppercase if (agentParams_p.isDefined("correlation")) { agentParams_p.get("correlation", correlation); correlation.upcase(); agentParams_p.define("correlation", correlation); } // Create one agent for each polarization if (mode.compare("tfcrop") == 0) { if (not agentParams_p.isDefined("correlation")) { // Default for tfcrop correlation = "ABS_ALL"; if (datacolumn.compare("FLOAT_DATA") == 0) correlation = "REAL_ALL"; agentParams_p.define("correlation", correlation); } if (dbg){ cout << "mode="<<mode<<" correlation="<<correlation<<endl; } // Is the expression polarization an ALL? if (isExpressionPolarizationAll(correlation)) { // Get the complex unitary function (ABS, NORM, REAL, IMAG, ARG) String function = getExpressionFunction(correlation); // Get all the polarizations in the MS std::vector<String> *allpol = fdh_p->corrProducts_p; for (size_t i=0; i < allpol->size(); i++){ // compose the full expression String func = function; String pol = allpol->at(i); String exp = func.append("_"); exp = func.append(pol); // Save the record to a list of agents agentParams_p.define("correlation", exp); String name = agent_name; name = name.append("_"); name = name.append(pol); agentParams_p.define("agentname", name); listOfAgents.push_back(agentParams_p); } } } // Check if extend agent should be added to the list later if (mode.compare("tfcrop") == 0 or mode.compare("rflag") == 0){ if (agentParams_p.isDefined("extendflags")) agentParams_p.get("extendflags", extendflags_p); } if (listOfAgents.size() > 0) { // add the agent(s) to the list for (size_t i=0; i < listOfAgents.size(); i++) { agents_config_list_p.push_back(listOfAgents.at(i)); } } else { agents_config_list_p.push_back(agentParams_p); } if (dbg){ for (size_t i=0; i < agents_config_list_p.size(); i++) { ostringstream os; agents_config_list_p.at(i).print(os); String str(os.str()); cout << str << endl; } } // Keep a copy of the vector of Agents parameters agents_config_list_copy_p = agents_config_list_p; // Parse the parameters of the extend agent if ((mode.compare("tfcrop") == 0 or mode.compare("rflag") == 0) and extendflags_p==true){ os << LogIO::NORMAL << "Will extend the flags after " << mode << "; extendflags=true" << LogIO::POST; extendflags_p = false; parseExtendParameters( "", // field "", // spw "", // array "", // feed "", // scan "", // antenna "", // uvrange "", // timerange "", // correlation "", // intent "", // observation max_p, // ntime combinescans_p, // combinescans true, // extendpols 50.0, // growtime 80.0 // growfreq // false, // growaround // false, // flagneartime // false, // flagnearfreq // true // apply ); } return true; } // --------------------------------------------------------------------- // AgentFlagger::initAgents // Initialize the Agents // Call parseAgentParameters and selectData first // --------------------------------------------------------------------- bool AgentFlagger::initAgents() { LogIO os(LogOrigin("AgentFlagger",__FUNCTION__)); if (! fdh_p){ os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST; return false; } if (dataselection_p.empty()){ os << LogIO::SEVERE << "There is no MS Selection available." << LogIO::POST; return false; } if (agents_config_list_p.empty()){ return false; } os<< LogIO::DEBUG1<< "There are initially "<< agents_config_list_p.size()<< " agents in the list"<<LogIO::POST; size_t list_size = agents_config_list_p.size(); // Send the logging of the re-applying agents to the debug // as we are only interested in seeing the unapply action (flagcmd) Bool retstate = true; // Loop through the vector of agents for (size_t i=0; i < list_size; i++) { // Get agent record Record agent_rec = agents_config_list_p[i]; if (dbg){ os<< LogIO::NORMAL<< "Record["<<i<<"].nfields()="<<agent_rec.nfields()<<LogIO::POST; ostringstream os; agent_rec.print(os); String str(os.str()); cout << str << endl; } // Get the mode String mode; agent_rec.get("mode", mode); // If clip agent is mixed with other agents and time average is true, skip it if ((mode.compare("clip") == 0 and list_size > 1) or (mode.compare("rflag") == 0 and list_size > 2) or (mode.compare("tfcrop") == 0 and list_size > 2)) { Bool tavg = false; int exists = agent_rec.fieldNumber ("timeavg"); if (exists >= 0) agent_rec.get("timeavg", tavg); if (tavg){ os << LogIO::WARN << "Cannot have " << mode <<" mode with timeavg/channelavg=true in list mode" << LogIO::POST; continue; } } // Set the new time interval only once if (!timeset_p and (mode.compare("tfcrop") == 0 or mode.compare("extend") == 0 or mode.compare("rflag") == 0)) { fdh_p->setTimeInterval(max_p); timeset_p = true; } // Change the new iteration approach only once if (!iterset_p and (mode.compare("tfcrop") == 0 or mode.compare("extend") == 0 or mode.compare("rflag") == 0 or mode.compare("display") == 0)) { if (combinescans_p) fdh_p->setIterationApproach(FlagDataHandler::COMBINE_SCANS_MAP_ANTENNA_PAIRS_ONLY); else fdh_p->setIterationApproach(FlagDataHandler::COMPLETE_SCAN_MAP_ANTENNA_PAIRS_ONLY); iterset_p = true; } // Agent's name String agent_name; agent_rec.get("agentname",agent_name); FlagAgentBase *fa = 0; try { // CAS-3943 (flagdata seg-faults when non-existent data column is to be read) Bool createAgent = true; if (((mode.compare("tfcrop") == 0 or mode.compare("rflag") == 0 or mode.compare("clip") == 0 or mode.compare("display") == 0 or mode.compare("antint") == 0)) and (agent_rec.fieldNumber ("datacolumn") >= 0)) { String datacolumn; agent_rec.get("datacolumn", datacolumn); datacolumn.upcase(); if ((datacolumn.compare("CORRECTED") == 0) or (datacolumn.compare("RESIDUAL") == 0)) { createAgent = fdh_p->checkIfColumnExists(MS::columnName(MS::CORRECTED_DATA)); if (!createAgent) { os << LogIO::WARN << "Agent " << agent_name << " cannot be created, necessary CORRECTED_DATA column is not available" << LogIO::POST; os << LogIO::WARN << "If wishing to flag residuals, use datacolumn=\'RESIDUAL_DATA\', which will calculate DATA-MODEL" << LogIO::POST; } } } // Create this agent if the necessary columns are available if (createAgent) { FlagAgentBase *tfa = FlagAgentBase::create(fdh_p, agent_rec); fa = tfa; } } catch(AipsError x) { fa = 0; // Send out a useful message, and stop adding agents to the list. // All valid agents before the problematic one, will remain in agents_list_p // A subsequent call to initAgents() will add to the list. ostringstream oss; agent_rec.print(oss); String recstr(oss.str()); while(recstr.contains('\n')) { recstr = recstr.replace(recstr.index('\n'),1,", "); } os << LogIO::WARN << "Cannot create agent "<< agent_name << ": " << x.getMesg() << LogIO::POST; os << LogIO::DEBUG1<< "Input parameters : " << recstr << LogIO::POST; continue; } if (fa == 0){ os << LogIO::WARN << "Agent "<< agent_name<< " is NULL. Skipping it."<<LogIO::POST; continue; } /* // Get the last summary agent to list the results back to the task if (mode.compare("summary") == 0) { summaryAgent_p = (FlagAgentSummary *) fa; } */ // Get the display agent. #if ! defined(WITHOUT_DBUS) if (mode.compare("display") == 0){ displayAgent_p = (FlagAgentDisplay *) fa; } #endif // Add the agent to the FlagAgentList agents_list_p.push_back(fa); } os << LogIO::NORMAL << "There are "<< agents_list_p.size()<<" valid agents in list"<< LogIO::POST; // Clear the list so that this method cannot be called twice agents_config_list_p.clear(); if (agents_list_p.size() < 1) retstate = false; return retstate; } // --------------------------------------------------------------------- // AgentFlagger::run // Run the agents // It assumes that initAgents has been called first // --------------------------------------------------------------------- Record AgentFlagger::run(Bool writeflags, Bool sequential) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); if (! fdh_p){ os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST; return Record(); } if (agents_list_p.empty()) { os << LogIO::SEVERE << "There is no agent to run in list"<< LogIO::POST; return Record(); } // Use the maximum ntime of the list os << LogIO::DEBUG2 << "ntime for all agents will be "<< max_p << LogIO::POST; os << LogIO::DEBUG2 << "combinescans for all agents will be "<< combinescans_p << LogIO::POST; // Report to return FlagReport combinedReport; // Generate the iterators // It will iterate through the data to evaluate the necessary memory // and get the START and STOP values of the scans for the quack agent fdh_p->generateIterator(); agents_list_p.start(); os << LogIO::DEBUGGING << "Size of agent's list is " << agents_list_p.size()<< LogIO::POST; // iterate over chunks while (fdh_p->nextChunk()) { // iterate over visBuffers while (fdh_p->nextBuffer()) { // Apply or unapply the flags, in sequential or in parallel. // By default it is set to true, which will preserve the order // in which the agents are added to the list. agents_list_p.apply(sequential); // Flush flags to MS if (writeflags) fdh_p->flushFlags(); } // Print the chunk summary stats if (fdh_p->summarySignal()) { agents_list_p.chunkSummary(); fdh_p->progressCounts_p = 0; } } // Print the MS summary stats agents_list_p.tableSummary(); if (writeflags) os << LogIO::NORMAL << "=> " << "Writing flags to the MS" << LogIO::POST; agents_list_p.terminate(); agents_list_p.join(); // Gather the display reports from all agents combinedReport = agents_list_p.gatherReports(); // Send reports to display agent #if ! defined(WITHOUT_DBUS) if (displayAgent_p) displayAgent_p->displayReports(combinedReport); #endif agents_list_p.clear(); return combinedReport; } // --------------------------------------------------------------------- // AgentFlagger::isExpressionPolarizationAll // Returns true if expression contains a polarization ALL // // --------------------------------------------------------------------- bool AgentFlagger::isExpressionPolarizationAll(String correlation) { if (correlation.find("ALL") == string::npos){ return false; } return true; } // --------------------------------------------------------------------- // AgentFlagger::getExpressionFunction // Get the unitary function of a polarization expression // returns a String with the function name // // --------------------------------------------------------------------- String AgentFlagger::getExpressionFunction(String expression) { String func; // Parse complex unitary function if (expression.find("REAL") != string::npos) { func = "REAL"; } else if (expression.find("IMAG") != string::npos) { func = "IMAG"; } else if (expression.find("ARG") != string::npos) { func = "ARG"; } else if (expression.find("ABS") != string::npos) { func = "ABS"; } else if (expression.find("NORM") != string::npos) { func = "NORM"; } else { return ""; } return func; } // --------------------------------------------------------------------- // AgentFlagger::getMax // Get the maximum between two values and // assign it to the max_p class member. // --------------------------------------------------------------------- void AgentFlagger::getMax(Double value) { if (value > max_p) max_p = value; return; } // --------------------------------------------------------------------- // AgentFlagger::getFlagVersionList // Get the flag versions list from the file FLAG_VERSION_LIST in the // MS directory // // --------------------------------------------------------------------- bool AgentFlagger::getFlagVersionList(Vector<String> &verlist) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__, WHERE)); try { if (! fdh_p){ os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST; return false; } verlist.resize(0); Int num; FlagVersion fv(fdh_p->getTableName(),"FLAG","FLAG_ROW"); Vector<String> vlist = fv.getVersionList(); num = verlist.nelements(); verlist.resize( num + vlist.nelements() + 1, true ); verlist[num] = String("\nMS : ") + fdh_p->getTableName() + String("\n"); for(Int j=0; j<(Int)vlist.nelements(); j++) verlist[num+j+1] = vlist[j]; } catch (AipsError x) { os << LogIO::SEVERE << "Could not get Flag Version List : " << x.getMesg() << LogIO::POST; return false; } return true; } // --------------------------------------------------------------------- // AgentFlagger::printFlagSelection // Print the flag versions list // // --------------------------------------------------------------------- bool AgentFlagger::printFlagSelections() { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__, WHERE)); // Use the copy of the agent records list because the original // was deallocated in the init() method. if (! agents_config_list_copy_p.empty()) { os << "Current list of agents : " << LogIO::POST; // TODO: loop through list // Duplicate the vector... ??? for (size_t i=0; i < agents_config_list_copy_p.size(); i++) { ostringstream out; Record agent_rec; agent_rec = agents_config_list_copy_p.at(i); agent_rec.print(out); os << out.str() << LogIO::POST; } if (dbg) cout << "Size of original list " << agents_config_list_copy_p.size() << endl; } else os << " No current agents " << LogIO::POST; return true; } // --------------------------------------------------------------------- // AgentFlagger::saveFlagVersion // Save the flag version // // --------------------------------------------------------------------- bool AgentFlagger::saveFlagVersion(String versionname, String comment, String merge) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__, WHERE)); if (! fdh_p){ os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST; return false; } FlagVersion fv(fdh_p->getTableName(),"FLAG","FLAG_ROW"); fv.saveFlagVersion(versionname, comment, merge); return true; } // --------------------------------------------------------------------- // AgentFlagger::restoreFlagVersion // Restore the flag version // // --------------------------------------------------------------------- bool AgentFlagger::restoreFlagVersion(Vector<String> versionname, String merge) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__, WHERE)); if (! fdh_p){ os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST; return false; } try { FlagVersion fv(fdh_p->getTableName(),"FLAG","FLAG_ROW"); for(Int j=0;j<(Int)versionname.nelements();j++) fv.restoreFlagVersion(versionname[j], merge); } catch (AipsError x) { os << LogIO::SEVERE << "Could not restore Flag Version : " << x.getMesg() << LogIO::POST; return false; } return true; } // --------------------------------------------------------------------- // AgentFlagger::deleteFlagVersion // Delete the flag version // // --------------------------------------------------------------------- bool AgentFlagger::deleteFlagVersion(Vector<String> versionname) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__, WHERE)); if (! fdh_p){ os << LogIO::SEVERE << "There is no MS attached. Please run af.open first." << LogIO::POST; return false; } try { FlagVersion fv(fdh_p->getTableName(),"FLAG","FLAG_ROW"); for(Int j=0;j<(Int)versionname.nelements();j++) fv.deleteFlagVersion(versionname[j]); } catch (AipsError x) { os << LogIO::SEVERE << "Could not delete Flag Version : " << x.getMesg() << LogIO::POST; return false; } return true; } // --------------------------------------------------------------------- // AgentFlagger::isModeValid // Check if mode is valid. // Return false if not in the list // --------------------------------------------------------------------- bool AgentFlagger::isModeValid(String mode) { bool ret; // The validation depends on the type of input if (isMS_p) { if (mode.compare("manual") == 0 or mode.compare("clip") == 0 or mode.compare("quack") == 0 or mode.compare("shadow") == 0 or mode.compare("elevation") == 0 or mode.compare("tfcrop") == 0 or mode.compare("extend") == 0 or mode.compare("rflag") == 0 or mode.compare("antint") == 0 or mode.compare("unflag") == 0 or mode.compare("summary") == 0 or mode.compare("display") == 0) { ret = true; } else ret = false; } else { if (mode.compare("manual") == 0 or mode.compare("clip") == 0 or mode.compare("quack") == 0 or mode.compare("tfcrop") == 0 or mode.compare("extend") == 0 or mode.compare("rflag") == 0 or mode.compare("antint") == 0 or mode.compare("unflag") == 0 or mode.compare("summary") == 0 or mode.compare("display") == 0) { ret = true; } else ret = false; } return ret; } Bool AgentFlagger::validateDataColumn(String datacol) { String datacolumn = ""; Bool ret = false; datacol.upcase(); LogIO os(LogOrigin("AgentFlagger", __FUNCTION__, WHERE)); // The validation depends on the type of input if (isMS_p) { if (datacol.compare("DATA") == 0) datacolumn = "DATA"; else if(datacol.compare("CORRECTED") == 0) datacolumn = "CORRECTED_DATA"; else if(datacol.compare("MODEL") == 0) datacolumn = "MODEL_DATA"; else if(datacol.compare("RESIDUAL") == 0) datacolumn = "RESIDUAL"; else if(datacol.compare("RESIDUAL_DATA") == 0) datacolumn = "RESIDUAL_DATA"; else if(datacol.compare("FLOAT_DATA") == 0) datacolumn = "FLOAT_DATA"; else if(datacol.compare("WEIGHT_SPECTRUM") == 0) datacolumn = "WEIGHT_SPECTRUM"; else if(datacol.compare("WEIGHT") == 0) datacolumn = "WEIGHT"; else datacolumn = ""; } else { // cal tables if (datacol.compare("FPARAM") == 0 or (datacol.compare("CPARAM") == 0) or (datacol.compare("SNR") == 0)) datacolumn = datacol; } // Check if requested column exist. Residual is calculated later from CORRECTED-MODEL if (datacolumn.compare("RESIDUAL") == 0 or datacolumn.compare("RESIDUAL_DATA") == 0){ ret = true; } else if (fdh_p->checkIfColumnExists(datacolumn)) ret = true; return ret; } // --------------------------------------------------------------------- // AgentFlagger::parseManualParameters // Parse data selection parameters and specific manual parameters // // --------------------------------------------------------------------- bool AgentFlagger::parseManualParameters(String field, String spw, String array, String feed, String scan, String antenna, String uvrange, String timerange, String correlation, String intent, String observation, Bool autocorr, Bool apply) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); // Default values for some parameters String mode = "manual"; String agent_name = "Manual"; // Create a record with the parameters Record agent_record = Record(); agent_record.define("mode", mode); agent_record.define("spw", spw); agent_record.define("scan", scan); agent_record.define("field", field); agent_record.define("antenna", antenna); agent_record.define("timerange", timerange); agent_record.define("correlation", correlation); agent_record.define("intent", intent); agent_record.define("feed", feed); agent_record.define("array", array); agent_record.define("uvrange", uvrange); agent_record.define("observation", observation); agent_record.define("apply", apply); agent_record.define("agentname", agent_name); agent_record.define("autocorr", autocorr); // Call the main method parseAgentParameters(agent_record); return true; } // --------------------------------------------------------------------- // AgentFlagger::parseClipParameters // Parse data selection parameters and specific clip parameters // // --------------------------------------------------------------------- bool AgentFlagger::parseClipParameters(String field, String spw, String array, String feed, String scan, String antenna, String uvrange, String timerange, String correlation, String intent, String observation, String datacolumn, Vector<Double> clipminmax, Bool clipoutside, Bool channelavg, casac::variant chanbin, Bool timeavg, String timebin, Bool clipzeros, Bool apply) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); // Default values for some parameters String mode = "clip"; String agent_name = "Clip"; // Create a record with the parameters Record agent_record = Record(); agent_record.define("mode", mode); agent_record.define("spw", spw); agent_record.define("scan", scan); agent_record.define("field", field); agent_record.define("antenna", antenna); agent_record.define("timerange", timerange); agent_record.define("correlation", correlation); agent_record.define("intent", intent); agent_record.define("feed", feed); agent_record.define("array", array); agent_record.define("uvrange", uvrange); agent_record.define("observation", observation); agent_record.define("apply", apply); agent_record.define("agentname", agent_name); agent_record.define("datacolumn", datacolumn); agent_record.define("clipminmax", clipminmax); agent_record.define("clipoutside", clipoutside); agent_record.define("channelavg", channelavg); if (chanbin.type() == casac::variant::INT) { agent_record.define("chanbin", chanbin.toInt()); } else if (chanbin.type() == casac::variant::INTVEC) { agent_record.define("chanbin", Array<Int>(chanbin.toIntVec())); } agent_record.define("timeavg", timeavg); agent_record.define("timebin", timebin); agent_record.define("clipzeros", clipzeros); // Call the main method parseAgentParameters(agent_record); return true; } // --------------------------------------------------------------------- // AgentFlagger::parseQuackParameters // Parse data selection parameters and specific quack parameters // // --------------------------------------------------------------------- bool AgentFlagger::parseQuackParameters(String field, String spw, String array, String feed, String scan, String antenna, String uvrange, String timerange, String correlation, String intent, String observation, String quackmode, Double quackinterval, Bool quackincrement, Bool apply) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); // Default values for some parameters String mode = "quack"; String agent_name = "Quack"; // Create a record with the parameters Record agent_record = Record(); agent_record.define("mode", mode); agent_record.define("spw", spw); agent_record.define("scan", scan); agent_record.define("field", field); agent_record.define("antenna", antenna); agent_record.define("timerange", timerange); agent_record.define("correlation", correlation); agent_record.define("intent", intent); agent_record.define("feed", feed); agent_record.define("array", array); agent_record.define("uvrange", uvrange); agent_record.define("observation", observation); agent_record.define("apply", apply); agent_record.define("agentname", agent_name); agent_record.define("quackmode", quackmode); agent_record.define("quackinterval", quackinterval); agent_record.define("quackincrement", quackincrement); // Call the main method parseAgentParameters(agent_record); return true; } // --------------------------------------------------------------------- // AgentFlagger::parseElevationParameters // Parse data selection parameters and specific elevation parameters // // --------------------------------------------------------------------- bool AgentFlagger::parseElevationParameters(String field, String spw, String array, String feed, String scan, String antenna, String uvrange, String timerange,String correlation, String intent, String observation, Double lowerlimit, Double upperlimit, Bool apply) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); // Default values for some parameters String mode = "elevation"; String agent_name = "Elevation"; // Create a record with the parameters Record agent_record = Record(); agent_record.define("mode", mode); agent_record.define("spw", spw); agent_record.define("scan", scan); agent_record.define("field", field); agent_record.define("antenna", antenna); agent_record.define("timerange", timerange); agent_record.define("correlation", correlation); agent_record.define("intent", intent); agent_record.define("feed", feed); agent_record.define("array", array); agent_record.define("uvrange", uvrange); agent_record.define("observation", observation); agent_record.define("apply", apply); agent_record.define("agentname", agent_name); agent_record.define("lowerlimit", lowerlimit); agent_record.define("upperlimit", upperlimit); // Call the main method parseAgentParameters(agent_record); return true; } // --------------------------------------------------------------------- // AgentFlagger::parseTfcropParameters // Parse data selection parameters and specific tfcrop parameters // // --------------------------------------------------------------------- bool AgentFlagger::parseTfcropParameters(String field, String spw, String array, String feed, String scan, String antenna, String uvrange, String timerange, String correlation, String intent, String observation, Double ntime, Bool combinescans, String datacolumn, Double timecutoff, Double freqcutoff, String timefit, String freqfit, Int maxnpieces, String flagdimension, String usewindowstats, Int halfwin, Bool extendflags, Bool apply, Bool channelavg, casac::variant chanbin, Bool timeavg, String timebin) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); // Default values for some parameters String mode = "tfcrop"; String agent_name = "Tfcrop"; // Create a record with the parameters Record agent_record = Record(); agent_record.define("mode", mode); agent_record.define("spw", spw); agent_record.define("scan", scan); agent_record.define("field", field); agent_record.define("antenna", antenna); agent_record.define("timerange", timerange); agent_record.define("correlation", correlation); agent_record.define("intent", intent); agent_record.define("feed", feed); agent_record.define("array", array); agent_record.define("uvrange", uvrange); agent_record.define("observation", observation); agent_record.define("apply", apply); agent_record.define("agentname", agent_name); agent_record.define("ntime", ntime); agent_record.define("combinescans", combinescans); agent_record.define("datacolumn", datacolumn); agent_record.define("timecutoff", timecutoff); agent_record.define("freqcutoff", freqcutoff); agent_record.define("timefit", timefit); agent_record.define("freqfit", freqfit); agent_record.define("maxnpieces", maxnpieces); agent_record.define("flagdimension", flagdimension); agent_record.define("usewindowstats", usewindowstats); agent_record.define("halfwin", halfwin); agent_record.define("extendflags", extendflags); agent_record.define("apply", apply); if (chanbin.type() == casac::variant::INT) { agent_record.define("chanbin", chanbin.toInt()); } else if (chanbin.type() == casac::variant::INTVEC) { agent_record.define("chanbin", Array<Int>(chanbin.toIntVec())); } agent_record.define("channelavg", channelavg); agent_record.define("timeavg", timeavg); agent_record.define("timebin", timebin); // Call the main method parseAgentParameters(agent_record); return true; } // --------------------------------------------------------------------- // AgentFlagger::parseAntIntParameters // Parse data selection parameters and specific antint parameters // // --------------------------------------------------------------------- bool AgentFlagger::parseAntIntParameters(String field, String spw, String array, String feed, String scan, String antenna, String uvrange, String timerange, String correlation, String intent, String observation, String antint_ref_antenna, Double minchanfrac, Bool verbose, Bool apply) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); // Mode specific values for some general parameters String mode = "antint"; String agent_name = "AntennaIntegrations"; // Create a record with the parameters for this mode Record agent_record = Record(); agent_record.define("mode", mode); agent_record.define("agentname", agent_name); agent_record.define("field", field); agent_record.define("spw", spw); agent_record.define("scan", scan); agent_record.define("antenna", antenna); agent_record.define("uvrange", uvrange); agent_record.define("timerange", timerange); agent_record.define("correlation", correlation); agent_record.define("intent", intent); agent_record.define("observation", observation); agent_record.define("antint_ref_antenna", antint_ref_antenna); agent_record.define("minchanfrac", minchanfrac); agent_record.define("verbose", verbose); agent_record.define("apply", apply); // Call the main method parseAgentParameters(agent_record); return true; } // --------------------------------------------------------------------- // AgentFlagger::parseExtendParameters // Parse data selection parameters and specific extend parameters // // --------------------------------------------------------------------- bool AgentFlagger::parseExtendParameters(String field, String spw, String array, String feed, String scan, String antenna, String uvrange, String timerange, String correlation, String intent, String observation, Double ntime, Bool combinescans, Bool extendpols, Double growtime, Double growfreq, Bool growaround, Bool flagneartime, Bool flagnearfreq, Bool apply) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); // Default values for some parameters String mode = "extend"; String agent_name = "Extend"; // Create a record with the parameters Record agent_record = Record(); agent_record.define("mode", mode); agent_record.define("spw", spw); agent_record.define("scan", scan); agent_record.define("field", field); agent_record.define("antenna", antenna); agent_record.define("timerange", timerange); agent_record.define("correlation", correlation); agent_record.define("intent", intent); agent_record.define("feed", feed); agent_record.define("array", array); agent_record.define("uvrange", uvrange); agent_record.define("observation", observation); agent_record.define("apply", apply); agent_record.define("agentname", agent_name); agent_record.define("ntime", ntime); agent_record.define("combinescans", combinescans); agent_record.define("extendpols", extendpols); agent_record.define("growtime", growtime); agent_record.define("growfreq", growfreq); agent_record.define("growaround", growaround); agent_record.define("flagneartime", flagneartime); agent_record.define("flagnearfreq", flagnearfreq); agent_record.define("apply", apply); // Call the main method parseAgentParameters(agent_record); return true; } // --------------------------------------------------------------------- // AgentFlagger::parseSummaryParameters // Parse data selection parameters and specific summary parameters // // --------------------------------------------------------------------- bool AgentFlagger::parseSummaryParameters(String field, String spw, String array, String feed, String scan, String antenna, String uvrange, String timerange, String correlation, String intent, String observation, Bool spwchan, Bool spwcorr, Bool basecnt, Bool fieldcnt, String name) { LogIO os(LogOrigin("AgentFlagger", __FUNCTION__)); // Default values for some parameters String mode = "summary"; String agent_name = "Summary"; // Create a record with the parameters Record agent_record = Record(); agent_record.define("mode", mode); agent_record.define("spw", spw); agent_record.define("scan", scan); agent_record.define("field", field); agent_record.define("antenna", antenna); agent_record.define("timerange", timerange); agent_record.define("correlation", correlation); agent_record.define("intent", intent); agent_record.define("feed", feed); agent_record.define("array", array); agent_record.define("uvrange", uvrange); agent_record.define("observation", observation); agent_record.define("agentname", agent_name); agent_record.define("spwchan", spwchan); agent_record.define("spwcorr", spwcorr); agent_record.define("basecnt", basecnt); agent_record.define("fieldcnt", fieldcnt); agent_record.define("name", name); // Call the main method parseAgentParameters(agent_record); return true; } } //#end casa namespace