//# AgentFlagger.h: this defines AgentFlagger //# Copyright (C) 2000,2001 //# Associated Universities, Inc. Washington DC, USA. //# //# This library is free software; you can redistribute it and/or modify it //# under the terms of the GNU Library General Public License as published by //# the Free Software Foundation; either version 2 of the License, or (at your //# option) any later version. //# //# This library is distributed in the hope that it will be useful, but WITHOUT //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public //# License for more details. //# //# You should have received a copy of the GNU Library General Public License //# along with this library; if not, write to the Free Software Foundation, //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA. //# //# Correspondence concerning AIPS++ should be addressed as follows: //# Internet email: aips2-request@nrao.edu. //# Postal address: AIPS++ Project Office //# National Radio Astronomy Observatory //# 520 Edgemont Road //# Charlottesville, VA 22903-2475 USA //# //# $Id$ #ifndef FLAGGING_FLAGGER_H #define FLAGGING_FLAGGER_H #include <iostream> #include <vector> #include <casa/Logging/LogIO.h> #include <casa/Arrays/Vector.h> #include <casa/Containers/Record.h> #include <casa/Quanta/Quantum.h> #include <flagging/Flagging/FlagVersion.h> #include <flagging/Flagging/FlagDataHandler.h> #include <flagging/Flagging/FlagMSHandler.h> #include <flagging/Flagging/FlagCalTableHandler.h> #include <flagging/Flagging/FlagAgentBase.h> #include <flagging/Flagging/FlagAgentSummary.h> #if ! defined(WITHOUT_DBUS) #include <flagging/Flagging/FlagAgentDisplay.h> #endif namespace casa { //# NAMESPACE CASA - BEGIN // <summary> // AgentFlagger: high-performance automated flagging // </summary> // <use visibility=global> // <reviewed reviewer="" date="" tests="" demos=""> // </reviewed> // <prerequisite> // <li> <linkto class="FlagDataHandler:description">FlagDataHandler</linkto> // <li> <linkto class="FlagMSHandlerr:description">FlagMSHandler</linkto> // <li> <linkto class="FlagCalTableHandler:description">FlagCalTableHandler</linkto> // </prerequisite> // // <etymology> // // </etymology> // // <synopsis> // AgentFlagger performs automated flagging operations on a measurement set or calibration // table. The class is constructed from an casacore::MS or cal table. After that, the run method may be used // to run any number of flagging agents. // </synopsis> // // <example> // <srcblock> // // The following is a typical way of using this class and its methods to perform flagging. // // // Open the casacore::MS or a calibration file and attach it to the tool. This will create an object of the // // FlagDataHandler (fdh) type. The constructor of the fdh takes three arguments, // // the casacore::MS or CAL table, the iteration approach to use and the time interval. Only the MS // // is mandatory to use. By default it will use the FlagDataHandler::SUB_INTEGRATION iteration // // approach and 0.0 seconds as the time interval. // // AgentFlagger *tf = new AgentFlagger(); // af->open('my.ms') // // // Select the data where to flag. If left blank, the whole casacore::MS will be selected. This step // // will use the casacore::MS Selection class. There are two methods to perform the selection. One takes // // a casacore::Record of the parameters, the other takes the individual parameters as arguments. // // // 1) First method: // casacore::String spw = "0:1~10"; // casacore::String scan = "1"; // casacore::Record selection = casacore::Record(); // selection.define("spw", spw); // selection.define("scan", scan); // af->selectData(selection); // // // 2) Second method: // af->selectData(spw=spw, scan=scan); // // // Now it is time to build a list of the agents that we want to run to process the data. This // // step will create a list of all the agents that will be executed to flag/unflag the data. // // This method can be called multiple times. Every call should contain the desired parameters of // // the agent and optionally data selection parameters. When data selection parameters are present, // // the agent will loop through only that portion of the data. // // // This method will check if the requested agent (mode) is known from the following list // // (manual, clip, quack, shadow, elevation, tfcrop, rflag, extend, unflag, summary, and antint). If // // empty or unknown, it will give a warning and return. // // // If any tfcrop, rflag or extend mode is present, this method will calculate the maximum value // // of time interval (ntime) from these agents. The maximum value will be used for all agents in // // the list. // // // A similar situation will happen with the combinescans parameter. If any of the combinescans is // // true, it will be taken as true to all agents. // // // Async I/O will be activated if any of the modes clip, tfcrop or rflag is requested. Also for // // these three modes, there will be a call to a function that will validate the requested // // datacolumn parameter. It will detect if the input is an casacore::MS or a cal table and validate the // // column. The default is the DATA column. If the input is a cal table, the function will // // first check if FPARAM is available, then CPARAM. If none of them is available it will return // // false and the agent will not be created. // // // Only for the tfcrop agent, if a correlation ALL is requested, this method will create one // // agent for each available polarization in the MS. For example, if the casacore::MS contains polarizations // // XX and YY and the parameter is correlation="ABS_ALL", then there will be two tfcrop agents, // // one with correlation="ABS_XX" and the other with correlation="ABS_YY". The apply parameter // // is set by default to true to apply the flags. // // casacore::Record agent_pars = casacore::Record(); // agent_pars.define("mode", "clip"); // agent_pars.define("clipzeros", true); // agent_pars.define("apply", true); // af->parseAgentParameters(agent_pars); // // casacore::Record agent_pars = casacore::Record(); // agent_pars.define("mode", "manual"); // agent_pars.define("autocorr", true); // af->parseAgentParameters(agent_pars); // // casacore::Record agent_pars = casacore::Record(); // agent_pars.define("mode", "summary"); // agent_pars.define("basecnt", true); // af->parseAgentParameters(agent_pars); // // // There are convenience functions to parse the agent's parameters, one specific for each agent. // // The above calls can also be done using these functions instead. // // af->parseClipParameters(clipzeros=true, apply=true); // af->parseManualParameters(autocorr=true); // af->parseSummaryParameters(basecnt=true); // // // In either one of the cases, three agents will be created. // // // // NOTE: it is possible to add multiple summary agents to the list and gather a list of summary // // reports when executing the tool. // // // We need to initialize the agents, which // // will call the constructor of each one of them and set the parameters that were given in the previous // // calls. Some basic checks will be performed at this stage for types and values of the parameters. // // // If any tfcrop, rflag, extend or display agent is in the list, the iteration approach will be // // set to a different value depending on whether combinescans is true or not. When true, the // // iteration approach will be set to FlagDataHandler::COMBINE_SCANS_MAP_ANTENNA_PAIRS_ONLY, otherwise // // to FlagDataHandler::COMPLETE_SCAN_MAP_ANTENNA_PAIRS_ONLY. // // // This method will create agents and add them to a FlagAgentList. If for any reason, the call to // // FlagAgentBase::create(fdh_p, agent_rec) fails, an error message will be displayed. Any agents previously // // added to the FlagAgentList will remain there. A subsequent call to this method can be done to add // // more agents to the same FlagAgentList. // // af->initAgents(); // // // Before next step which will write the new flags, it is advisable to create a backup of // // the current flags in the MS. // // af.saveflagversion(versionname='backup_before_manual_1', // comment='Backup of flags before running manual',merge='replace') // // // The next step in the chain is to actually process the flags and write them or // // not to the MS. The run method takes two parameters, writeflags and sequential. // // The parameter writeflags controls whether to write the flags or not to the MS. // // By default it is set to true. Setting writeflags to false is useful when one // // wants to run the tool together with the display agent to see what is going to be // // flagged before deciding to write or not to the MS. The sequential parameter // // controls if the order of the agent's list needs to be preserved or not. If set to false, // // the order will not be preserved and the framework may execute the agent's list in parallel. // // By default sequential is set to true. // // // The run method gathers several reports, depending on which agents are run. The display and summary agents // // produce reports that can be retrieved from calling the run method. The reports are returned via a Record // // that may contain multiple reports at the same time. // casacore::Record myReports; // myReports = af->run(); // // // To destroy the tool, call a method to execute the destructor. // // af->done(); // // </srcblock> // </example> // // <motivation> // To flag data using different algorithms. // </motivation> // class AgentFlagger { protected: static casacore::LogIO os; // variables used to initialize the FlagDataHandler casacore::String msname_p; casacore::uShort iterationApproach_p; casacore::Double timeInterval_p; casacore::Bool isMS_p; // members to parse to selectData casacore::String spw_p; casacore::String scan_p; casacore::String field_p; casacore::String antenna_p; casacore::String timerange_p; casacore::String correlation_p; casacore::String intent_p; casacore::String feed_p; casacore::String array_p; casacore::String uvrange_p; casacore::String observation_p; casacore::Record dataselection_p; // agent's members casacore::String mode_p; casacore::Record agentParams_p; FlagAgentSummary *summaryAgent_p; casacore::Bool combinescans_p; casacore::Bool extendflags_p; // true if there are apply and unapply parameters in the list casacore::Bool mixed_p; // Display agent parameters #if ! defined(WITHOUT_DBUS) FlagAgentDisplay *displayAgent_p; #endif // variables for initAgents FlagDataHandler *fdh_p; std::vector<casacore::Record> agents_config_list_p; std::vector<casacore::Record> agents_config_list_copy_p; FlagAgentList agents_list_p; public: // default constructor AgentFlagger(); // destructor ~AgentFlagger(); // reset everything void done(); // configure the tool, open the MS bool open(casacore::String msname, casacore::Double ntime); // parse the data selection bool selectData(casacore::Record selrec); bool selectData(casacore::String field, casacore::String spw, casacore::String array, casacore::String feed, casacore::String scan, casacore::String antenna, casacore::String uvrange, casacore::String timerange, casacore::String correlation, casacore::String intent, casacore::String observation=""); // parse the parameters of the agent bool parseAgentParameters(casacore::Record agent_params); casacore::String getExpressionFunction(casacore::String expression); bool isExpressionPolarizationAll(casacore::String expression); // initialize the agents list bool initAgents(); // Run the tool and write the flags to the MS casacore::Record run(casacore::Bool writeflags, casacore::Bool sequential=true); // Flag backup methods bool printFlagSelections(); bool saveFlagVersion(casacore::String versionname, casacore::String comment, casacore::String merge); bool restoreFlagVersion(casacore::Vector<casacore::String> versionname, casacore::String merge); bool deleteFlagVersion(casacore::Vector<casacore::String> versionname); bool getFlagVersionList(casacore::Vector<casacore::String> &verlist); // Agent's specific parsing methods (for convenience only) // Parse parameters for manual bool parseManualParameters(casacore::String field, casacore::String spw, casacore::String array, casacore::String feed, casacore::String scan, casacore::String antenna, casacore::String uvrange, casacore::String timerange,casacore::String correlation, casacore::String intent, casacore::String observation, casacore::Bool autocorr, casacore::Bool apply); // Parse parameters for clip bool parseClipParameters(casacore::String field, casacore::String spw, casacore::String array, casacore::String feed, casacore::String scan, casacore::String antenna, casacore::String uvrange, casacore::String timerange,casacore::String correlation, casacore::String intent, casacore::String observation, casacore::String datacolumn, casacore::Vector<casacore::Double> clipminmax, casacore::Bool clipoutside, casacore::Bool channelavg, casac::variant chanbin, casacore::Bool timeavg, casacore::String timebin, casacore::Bool clipzeros, casacore::Bool apply); // Parse parameters for quack bool parseQuackParameters(casacore::String field, casacore::String spw, casacore::String array, casacore::String feed, casacore::String scan, casacore::String antenna, casacore::String uvrange, casacore::String timerange,casacore::String correlation, casacore::String intent, casacore::String observation, casacore::String quackmode, casacore::Double quackinterval, casacore::Bool quackincrement, casacore::Bool apply); // Parse parameters for elevation bool parseElevationParameters(casacore::String field, casacore::String spw, casacore::String array, casacore::String feed, casacore::String scan, casacore::String antenna, casacore::String uvrange, casacore::String timerange, casacore::String correlation, casacore::String intent, casacore::String observation, casacore::Double lowerlimit, casacore::Double upperlimit, casacore::Bool apply); // Parse parameters for tfcrop bool parseTfcropParameters(casacore::String field, casacore::String spw, casacore::String array, casacore::String feed, casacore::String scan, casacore::String antenna, casacore::String uvrange, casacore::String timerange, casacore::String correlation, casacore::String intent, casacore::String observation, casacore::Double ntime, casacore::Bool combinescans, casacore::String datacolumn, casacore::Double timecutoff, casacore::Double freqcutoff, casacore::String timefit, casacore::String freqfit, casacore::Int maxnpieces, casacore::String flagdimension, casacore::String usewindowstats, casacore::Int halfwin, casacore::Bool extendflags, casacore::Bool apply, casacore::Bool channelavg, casac::variant chanbin, casacore::Bool timeavg, casacore::String timebin); // Parse parameters for the 'antint' mode (antenna integrations) bool parseAntIntParameters(casacore::String field, casacore::String spw, casacore::String array, casacore::String feed, casacore::String scan, casacore::String antenna, casacore::String uvrange, casacore::String timerange, casacore::String correlation, casacore::String intent, casacore::String observation, casacore::String antint_ref_antenna, casacore::Double minchanfrac, casacore::Bool verbose, casacore::Bool apply); // Parse parameters for extend bool parseExtendParameters(casacore::String field="", casacore::String spw="", casacore::String array="", casacore::String feed="", casacore::String scan="", casacore::String antenna="", casacore::String uvrange="", casacore::String timerange="", casacore::String correlation="", casacore::String intent="", casacore::String observation="", casacore::Double ntime=0.0, casacore::Bool combinescans=false, casacore::Bool extendpols=true, casacore::Double growtime=50.0, casacore::Double growfreq=50.0, casacore::Bool growaround=false, casacore::Bool flagneartime=false, casacore::Bool flagnearfreq=false, casacore::Bool apply=true); // Parse parameters for summary bool parseSummaryParameters(casacore::String field, casacore::String spw, casacore::String array, casacore::String feed, casacore::String scan, casacore::String antenna, casacore::String uvrange, casacore::String timerange, casacore::String correlation, casacore::String intent, casacore::String observation, casacore::Bool spwchan, casacore::Bool spwcorr, casacore::Bool basecnt, casacore::Bool fieldcnt, casacore::String name); private: AgentFlagger(const AgentFlagger &) {}; AgentFlagger& operator=(const AgentFlagger &) {return *this;}; // Maximum between two numbers void getMax(casacore::Double value); // Check if mode is valid against a list of known modes bool isModeValid(casacore::String mode); casacore::Bool validateDataColumn(casacore::String datacol); // Sink used to store history casacore::LogSink logSink_p; // Debug message flag static const bool dbg; // Store the temporary maximum value casacore::Double max_p; // Helper members casacore::Bool timeset_p; casacore::Bool iterset_p; // casacore::Time average parameter in clip mode casacore::Bool timeAvg_p; }; } //# NAMESPACE CASA - END #endif