Commits

David Mehringer authored and Ville Suoranta committed 0b6f227d7bf Merge
Pull request #782: CAS-14235 new WS client task getantposalma

Merge in CASA/casa6 from CAS-14235 to master * commit '3ca5b63d630430a4c0f97da661ee3e3f6b196db8': (37 commits) doc corrections requested by akepley tweak of output data structure change output struct from list to dict added "caltype" to metadata fix up test add json.loads() to write to a file something easier for gencal to handle. Requested by Enrique. minor doc updates new doc updates doc fix Update component_to_test_map to add getantposalma add metaata to output file as python dict check point check point change default server to prod server, update tests so they all pass finish renaming task rename antposalma -> getantposalma check point rename remove obsolete files, add test area for antposalma attempt to figure out when to use prod server vs dev server ...

casatasks/src/private/task_getantposalma.py

Added
1 +from casatasks import casalog
2 +from casatools import quanta
3 +import certifi
4 +from datetime import datetime
5 +import json, os, shutil
6 +import ssl
7 +from urllib import request
8 +from urllib.error import HTTPError, URLError
9 +from urllib.parse import urlencode, urlparse
10 +
11 +
12 +def _is_valid_url_host(url):
13 + parsed = urlparse(url)
14 + return bool(parsed.netloc)
15 +
16 +
17 +def _query(url):
18 + myjson = None
19 + response = None
20 + try:
21 + context = ssl.create_default_context(cafile=certifi.where())
22 + with request.urlopen(url, context=context, timeout=400) as response:
23 + if response.status == 200:
24 + myjson = response.read().decode('utf-8')
25 + except HTTPError as e:
26 + casalog.post(
27 + f"Caught HTTPError: {e.code} {e.reason}: {e.read().decode('utf-8')}",
28 + "WARN"
29 + )
30 + except URLError as e:
31 + casalog.post(f"Caught URLError: {str(e)}", "WARN")
32 + except Exception as e:
33 + casalog.post(f"Caught Exception when trying to connect: {str(e)}", "WARN")
34 + return myjson
35 +
36 +
37 +def getantposalma(
38 + outfile='', overwrite=False, asdm='', tw='', snr=0, search='both_latest',
39 + hosts=['tbd1.alma.cl', 'tbd2.alma.cl']
40 +):
41 + r"""
42 +Retrieve antenna positions by querying ALMA web service.
43 +
44 +[`Description`_] [`Examples`_] [`Development`_] [`Details`_]
45 +
46 +
47 +Parameters
48 + - outfile_ (path='') - Name of output file to which to write retrieved antenna positions.
49 + - overwrite_ (bool=False) - Overwrite a file by the same name if it exists?
50 + - asdm_ (string='') - The associated ASDM name. Must be specified
51 + - tw_ (string='') - Optional time window in which to consider baseline measurements in the database, when calculating the antenna positions.
52 + - snr_ (float=0) - Optional signal-to-noise.
53 + - search_ (string='both_latest') - Search algorithm to use.
54 + - hosts_ (stringVec=['https://asa.alma.cl/uncertainties-service/uncertainties/versions/last/measurements/casa/']) - Priority-ranked list of hosts to query.
55 +
56 +
57 +
58 +
59 +.. _Description:
60 +
61 +Description
62 +
63 +.. warning:: **WARNING**: This task should be considered experimental
64 + since the values returned by the JAO service are in the process of
65 + being validated.
66 +
67 +This task retrieves ALMA antenna positions via a web service which runs
68 +on an ALMA-hosted server. The antenna positions are with respect to ITRF.
69 +The user must specify the value of the outfile parameter. This parameter
70 +is the name of the file to which the antenna positions will be written.
71 +This file can then be read by gencal so that it can use the most up to
72 +date antenna positions for the observation.
73 +
74 +The web service is described by the server development team and can be
75 +found `at this location <https://asw.alma.cl/groups/ASW/-/packages/843>`__.
76 +
77 +The input parameters are discussed in detail below.
78 +
79 +outfile is required to be specified. It is the name of the file to which to
80 +write antenna positions.
81 +
82 +overwrite If False and a file with the same name exists, and exception
83 +will be thrown. If true, an existing file with the same name will be
84 +overwriten.
85 +
86 +asdm is required to be specified. It is the associated ASDM UID in the
87 +form uid://A002/Xc02418/X29c8.
88 +
89 +tw is an optional parameter. It is time window in which the antenna positions
90 +are required, specified as a comma separated pair. Times are UTC and are
91 +expressed in YY-MM-DDThh:mm:ss.sss format. The end time must be later than
92 +the begin time.
93 +
94 +snr is an optional parameter. It is the signal-to-noise ratio. Antenna
95 +positions which have corrections less than this value will not be written.
96 +If not specified, positions of all antennas will be written.
97 +
98 +tw and search are optional parameters and are coupled as follows. search
99 +indicates the search algorithm to use to find the desired antenna positions.
100 +Supported values of this parameter at the time of writing are 'both_latest'
101 +and 'both_closest'. The task passes the value of the search parameter verbatim to
102 +the web service, meaning that users can take advantage of new search algorithms
103 +as the web service team brings them online. The default algorithm used is
104 +'both_latest'. In general, the search is limited in time to the specified
105 +value of tw (time window). However, in the case that tw is not specified, the
106 +following rules apply. For 'both_latest', the last updated position for each
107 +antenna within the specified time window, or, if tw is not specified, within
108 +30 days after the observation will be returned, taking into account snr if
109 +specified, if provided.
110 +
111 +For 'both_closest', if tw is not specified, the position
112 +of each antenna closest in time to the observation, within 30 days (before
113 +or after the observation) will be returned, subject to the value of snr if it
114 +is specified.
115 +
116 +hosts is a required parameter. It is a list of hosts to query, in order of
117 +priority, to obtain positions. The first server to respond with a valid result is
118 +the only one that is used. That response will be written and no additional
119 +hosts will be queried.
120 +
121 +The format of the returned file is a two element list encoded in json. The first
122 +element is a stringfied dictionary that contains antenna names as keys, with each
123 +value being a three element list of x, y, and z coordinates in ITRF. The second
124 +element is a dictionary containing various (possibly helpful) metadata that were
125 +used when the task was run. The following code may be used to load these data
126 +structures into python variables.
127 +
128 + ::
129 +
130 + import ast, json
131 + ...
132 + with open("outfile.json", "r") as f:
133 + antpos_str, md_dict = json.load(f)
134 + antpos_dict = ast.literal_eval(antpos_str)
135 +
136 +
137 +.. _Examples:
138 +
139 +Examples
140 + Get antenna positions which have positions with a signal-to-noise ratio
141 + greater than 5.
142 +
143 + ::
144 +
145 + getantposalma(
146 + outfile='my_ant_pos.json', asdm='valid ASDM name here', snr=5,
147 + hosts=['tbd1.alma.cl', 'tbd2.alma.cl']
148 + )
149 +
150 +
151 +.. _Development:
152 +
153 +Development
154 + No additional development details
155 +
156 +
157 +
158 +
159 +.. _Details:
160 +
161 +
162 +Parameter Details
163 + Detailed descriptions of each function parameter
164 +
165 +.. _outfile:
166 +
167 +| ``outfile (path='')`` - Name of output file to which to write antenna positions. If a file by this name already exists, it will be silently overwritten. The written file will be in JSON format.
168 +| default: none
169 +| Example: outfile='my_alma_antenna_positions.json'
170 +
171 +.. _overwrite:
172 +
173 +| ``overwrite (bool=False)`` - Overwrite a file by the same name if it exists? If False and a file
174 +| with the same name exists, and exception will be thrown.
175 +
176 +.. _asdm:
177 +
178 +| ``asdm (string='')`` - The associated ASDM name. Must be specified. The ASDM is not required to be on the file system; its value is simply passed to the web service.
179 +| default: ''
180 +| Example:asdm='uid://A002/X10ac6bc/X896d'
181 +
182 +.. _tw:
183 +
184 +| ``tw (string='')`` - Optional time window in which to consider baseline measurements in the database, when calculating the antenna positions. Format is of the form begin_time,end_time, where times must be specified in YYYY-MM-DDThh:mm:ss.sss format and end_time must be later than begin time. Times should be UTC.
185 +| Example: tw='2023-03-14T00:40:20,2023-03-20T17:58:20'
186 +
187 +.. _snr:
188 +
189 +| ``snr (float=0)`` - Optional signal-to-noise. Antenna positions which have corrections with S/N less than this value will not be retrieved nor written. If not specified, positions of all antennas will be written.
190 +| default: 0 (no snr constraint will be used)
191 +| Example: snr=5.0
192 +
193 +.. _search:
194 +
195 +| ``search (string='both_latest')`` - Search algorithm to use. Supported values are "both_latest" and "both_closest". For "both_latest", the last updated position for each antenna within 30 days after the observation will be returned, taking into account snr if specified. If provided, tw will override the 30 day default value. For "both_closest", the position of each antenna closest in time to the observation, within 30 days (before or after the observation) will be returned, subject to the value of snr if it is specified. If specified, the value of tw will override the default 30 days. The default algorithm to use will be "both_latest".
196 +| Example: search="both_closest"
197 +
198 +.. _hosts:
199 +
200 +| ``hosts (stringVec=['https://asa.alma.cl/uncertainties-service/uncertainties/versions/last/measurements/casa/'])`` - Priority-ranked list of hosts to query to obtain positions. Only one server that returns a list of antenna positions is required. That response will be written and no additional hosts will be queried.
201 +| Example: hosts=["server1.alma.cl", "server2.alma.cl"]
202 +
203 +
204 + """
205 + if not outfile:
206 + raise ValueError("Parameter outfile must be specified")
207 + md = {
208 + "caltype": "ALMA antenna positions",
209 + "description": "ALMA ITRF antenna positions in meters",
210 + "product_code": "antposalma",
211 + "outfile": outfile
212 + }
213 + if not overwrite and os.path.exists(outfile):
214 + raise RuntimeError(
215 + f"A file or directory named {outfile} already exists and overwrite "
216 + "is False, so exiting. Either rename the existing file or directory, "
217 + "change the value of overwrite to True, or both."
218 + )
219 + if not hosts:
220 + raise ValueError("Parameter hosts must be specified")
221 + if isinstance(hosts, list) and not hosts[0]:
222 + raise ValueError("The first element of the hosts list must be specified")
223 + md["hosts"] = hosts
224 + _qa = quanta()
225 + parms = {}
226 + if asdm:
227 + parms['asdm'] = asdm
228 + else:
229 + raise ValueError("parameter asdm must be specified")
230 + if tw:
231 + z = tw.split(",")
232 + if len(z) != 2:
233 + raise ValueError(
234 + "Parameter tw should contain exactly one comma that separates two times"
235 + )
236 + s0, s1 = z
237 + msg = "The correct format is of the form YYYY-MM-DDThh:mm:ss."
238 + try:
239 + t_start = _qa.quantity(_qa.time(s0, form="fits")[0])
240 + except Exception as e:
241 + raise ValueError(f"Begin time {s0} does not appear to have a valid format. {msg}")
242 + try:
243 + t_end = _qa.quantity(_qa.time(s1, form="fits")[0])
244 + except Exception as e:
245 + raise ValueError(f"End time {s1} does not appear to have a valid format. {msg}")
246 + if _qa.ge(t_start, t_end):
247 + raise ValueError(
248 + f"Parameter tw, start time ({z[0]}) must be less than end time ({z[1]})."
249 + )
250 + parms["tw"] = tw
251 + if snr < 0:
252 + raise ValueError(f"Parameter snr ({snr}) must be non-negative.")
253 + elif snr > 0:
254 + parms["snr"] = snr
255 + if search:
256 + parms['search'] = search
257 + qs = f"?{urlencode(parms)}"
258 + md.update(parms)
259 + antpos = None
260 + for h in hosts:
261 + if not _is_valid_url_host(h):
262 + raise ValueError(
263 + f'Parameter hosts: {h} is not a valid host expressed as a URL.'
264 + )
265 + url = f"{h}/{qs}"
266 + casalog.post(f"Trying {url} ...", "NORMAL")
267 + antpos = _query(url)
268 + if antpos:
269 + md["successful_url"] = url
270 + antpos = json.loads(antpos)
271 + break
272 + if not antpos:
273 + raise RuntimeError("All URLs failed to return an antenna position list.")
274 + if os.path.exists(outfile):
275 + if overwrite:
276 + if os.path.isdir(outfile):
277 + casalog.post(
278 + f"Removing existing directory {outfile} before writing new "
279 + "file of same name",
280 + "WARN"
281 + )
282 + shutil.rmtree(outfile)
283 + else:
284 + casalog.post(
285 + f"Removing existing file {outfile} before writing now file of "
286 + "same name",
287 + "WARN"
288 + )
289 + os.remove(outfile)
290 + else:
291 + raise RuntimeError(
292 + "Logic Error: shouldn't have gotten to this point with overwrite=False"
293 + )
294 + md["timestamp"] = str(datetime.now())
295 + with open(outfile, "w") as f:
296 + json.dump({"data": antpos, "metadata": md}, f)

Everything looks good. We'll let you know here if there's anything you should know about.

Add shortcut