Commits

Sandra M Castro authored ed1f53d4a74 Merge
Merge branch 'master' into CAS-13798

casatasks/src/private/jyperk.py

Modified
8 8 import string
9 9 from time import sleep
10 10 from urllib.error import HTTPError, URLError
11 11 from urllib.parse import urlencode
12 12 from urllib.request import urlopen
13 13
14 14 import certifi
15 15 import numpy as np
16 16
17 17 from casatasks import casalog
18 -from casatasks.private.sdutil import table_selector, table_manager, tool_manager
18 +from casatasks.private.sdutil import (table_manager, table_selector,
19 + tool_manager)
19 20 from casatools import measures
20 21 from casatools import ms as mstool
21 22 from casatools import msmetadata, quanta
22 23
23 24
24 25 # Jy/K DB part
25 26 def gen_factor_via_web_api(vis, spw='*',
26 27 endpoint='asdm',
27 28 timeout=180, retry=3, retry_wait_time=5):
28 29 """Generate factors via Jy/K Web API.
103 104 def _vis_to_uid(vis):
104 105 """Convert MS name like 'uid___A002_Xabcd_X012 into uid://A002/Xabcd/X012'.
105 106
106 107 Arguments:
107 108 vis {str} -- The file path of the visibility data.
108 109
109 110 Returns:
110 111 str -- Corresponding ASDM uid.
111 112 """
112 113 basename = os.path.basename(os.path.abspath(vis))
113 - pattern = '^uid___A[0-9][0-9][0-9]_X[0-9a-f]+_X[0-9a-f]+\.ms$'
114 + pattern = r'^uid___A[0-9][0-9][0-9]_X[0-9a-f]+_X[0-9a-f]+\.ms$'
114 115 if re.match(pattern, basename):
115 116 return basename.replace('___', '://').replace('_', '/').replace('.ms', '')
116 117 else:
117 118 raise RuntimeError('MS name is not appropriate for DB query: {}'.format(basename))
118 119
119 120
120 121 class InterpolationParamsGenerator():
121 122 """A class to generate required parameters for Jy/K Web API with interpolation.
122 123
123 124 Usage:
129 130 def get_params(cls, vis, spw='*'):
130 131 if spw == '':
131 132 spw = '*'
132 133
133 134 if spw == '*':
134 135 spw = cls._get_available_spw(vis, spw)
135 136
136 137 params = {}
137 138
138 139 science_windows = cls._get_science_windows(vis, spw)
139 - timerange, antenna_names, basebands, mean_freqs, spwnames = cls._extract_msmetadata(science_windows, vis)
140 + timerange, antenna_names, basebands, mean_freqs, spwnames = \
141 + cls._extract_msmetadata(science_windows, vis)
140 142
141 143 mean_freqs = cls._get_mean_freqs(vis, science_windows)
142 144 bands = Bands.get(science_windows, spwnames, mean_freqs, vis)
143 145
144 146 params['date'] = cls._mjd_to_datestring(timerange['begin'])
145 147 params['temperature'] = cls._get_mean_temperature(vis)
146 148 params.update(cls._get_aux_params())
147 149
148 150 for antenna_id, antenna_name in enumerate(antenna_names):
149 151 params['antenna'] = antenna_name
173 175 spwnames = msmd.namesforspws(science_windows)
174 176
175 177 return timerange, antenna_names, basebands, mean_freqs, spwnames
176 178
177 179 @staticmethod
178 180 def _get_available_spw(vis, spw):
179 181 science_windows = InterpolationParamsGenerator._get_science_windows(vis, spw=spw)
180 182 with tool_manager(vis, msmetadata) as msmd:
181 183 spwnames = msmd.namesforspws(science_windows)
182 184
183 - spw = ','.join(map(str, [i for i, name in enumerate(spwnames) if not name.startswith('WVR')]))
185 + spw = ','.join(
186 + map(str, [i for i, name in enumerate(spwnames)
187 + if not name.startswith('WVR')]))
184 188 return spw
185 189
186 190 @staticmethod
187 191 def _mjd_to_datestring(epoch):
188 192 me = measures()
189 193 qa = quanta()
190 194
191 195 if epoch['refer'] != 'UTC':
192 196 try:
193 197 epoch = me.measure(epoch, 'UTC')
236 240 def get(cls, science_windows, spwnames, mean_freqs, vis):
237 241 """Return all bands corresponding to the 'science_window' given in the input.
238 242
239 243 First the method scan 'spwnames', if the band can be detect, the method will
240 244 adopt this value. In other case, the method compare the freq with the 'mean_freqs'
241 245 at which the band was detect, the method detect the band from the frequencies
242 246 that are closest to the result.
243 247 """
244 248 bands = cls._extract_bands_from_spwnames(science_windows, spwnames)
245 249 mean_freqs_with_undetected_band = cls._filter_mean_freqs_with_undetected_band(
246 - science_windows, spwnames, mean_freqs)
250 + science_windows, spwnames, mean_freqs)
247 251 if len(mean_freqs_with_undetected_band) > 0:
248 252 bands.update(
249 253 cls._detect_bands_from_mean_freqs(mean_freqs_with_undetected_band, vis)
250 254 )
251 255 return bands
252 256
253 257 @staticmethod
254 258 def _extract_bands_from_spwnames(science_windows, spwnames):
255 259 """Extract bands that contain band information in the spwname.
256 260
355 359 science_spw = list(np.intersect1d(
356 360 msmd.almaspws(tdm=True, fdm=True),
357 361 msmd.spwsforintent('OBSERVE_TARGET#ON_SOURCE')
358 362 ))
359 363 science_dd = [msmd.datadescids(spw=i)[0] for i in science_spw]
360 364
361 365 return science_dd
362 366
363 367 @staticmethod
364 368 def _query_rows(vis, science_dd, stateid, antenna_id):
365 - query = f'ANTENNA1=={antenna_id}&&ANTENNA2=={antenna_id}&&DATA_DESC_ID=={science_dd[0]}&&STATE_ID IN {list(stateid)}'
369 + query = f'ANTENNA1=={antenna_id}&&ANTENNA2=={antenna_id}&&DATA_DESC_ID=={science_dd[0]}' + \
370 + f'&&STATE_ID IN {list(stateid)}'
366 371 with table_selector(vis, query) as tb:
367 372 rows = tb.rownumbers()
368 373
369 374 return rows
370 375
371 376 @staticmethod
372 377 def _calc_elevation_mean(rows, vis):
373 378 elevations = []
374 379 qa = quanta()
375 380
400 405 manager = RequestsManager(client)
401 406 manager.get(params)
402 407 """
403 408
404 409 def __init__(self, client):
405 410 """Set client."""
406 411 self.client = client
407 412
408 413 def get(self, params):
409 414 """Get the responses of the Jy/K DB."""
410 - dataset = [{'response': self.client.get(param.param), 'aux': param.subparam} for param in params]
415 + dataset = [
416 + {'response': self.client.get(param.param), 'aux': param.subparam}
417 + for param in params]
411 418 return self._filter_success_is_true(dataset)
412 419
413 420 def _filter_success_is_true(self, dataset):
414 421 return [data for data in dataset if data['response']['success']]
415 422
416 423
417 424 class JyPerKDatabaseClient():
418 425 """A class to get values from Jy/K Web API.
419 426
420 427 The Jy/K Web API address is 'https://asa.alma.cl/science/jy-kelvins'. The address
430 437 endpoint {str} -- The endpoint of Jy/K DB Web API to access. Options are
431 438 'asdm' (default), 'model-fit', 'interpolation'.
432 439 timeout {int} --- Maximum waiting time [sec] for the Web API access, defaults
433 440 to 180 sec.
434 441 retry {int} -- Number of retry when the Web API access fails, defaults to 3
435 442 times.
436 443 retry_wait_time {int} -- Waiting time [sec] until next query when the Web API
437 444 access fails, defaults to 5 sec.
438 445 """
439 446 assert endpoint in ['asdm', 'model-fit', 'interpolation'], \
440 - 'The JyPerKDatabaseClient class requires one of endpoint: asdm, model-fit or interpolation'
447 + 'The JyPerKDatabaseClient class requires one of endpoint: ' \
448 + 'asdm, model-fit or interpolation'
441 449 self.web_api_url = self._generate_web_api_url(endpoint)
442 450 self.timeout = timeout
443 451 self.retry = retry
444 452 self.retry_wait_time = retry_wait_time
445 453
446 454 def get(self, param):
447 455 """Get the Web API response.
448 456
449 457 Arguments:
450 458 param {dict} -- The parameters used in the Web API.
499 507 msg = 'Failed to load URL: {0}\n'.format(url) \
500 508 + 'Error Message: URLError(Reason="{0}")\n'.format(e.reason)
501 509 casalog.post(msg)
502 510 return {'status': 'URLError', 'err_msg': msg}
503 511 except socket_timeout as e: # not connect
504 512 msg = 'Failed to load URL: {0}\n'.format(url) \
505 513 + 'Error Message: URLError(Reason="{0}")\n'.format(e)
506 514 casalog.post(msg)
507 515 return {'status': 'URLError', 'err_msg': msg}
508 516
509 -
510 517 def _try_to_get_response(self, url):
511 518 casalog.post(f'Accessing Jy/K DB: request URL is "{url}"')
512 519 for i in range(self.retry):
513 520 response_with_tag = self._retrieve(url)
514 521 if response_with_tag['status'] == 'Success':
515 - casalog.post(f'Got a response successfully')
522 + casalog.post('Got a response successfully')
516 523 return response_with_tag['body']
517 -
524 +
518 525 if i < self.retry - 1:
519 526 casalog.post(response_with_tag['err_msg'])
520 - casalog.post(f'Sleeping for {str(self.retry_wait_time)} seconds because the request failed')
527 + casalog.post(
528 + f'Sleeping for {str(self.retry_wait_time)} seconds because the request failed')
521 529 sleep(self.retry_wait_time)
522 530 casalog.post(f'Retry to access Jy/K DB ({str(i + 2)}/{str(self.retry)})')
523 531
524 532 if response_with_tag['status'] != 'Success':
525 533 raise RuntimeError(response_with_tag['err_msg'])
526 534
527 535 def _convert_to_json(self, response):
528 536 try:
529 537 return json.loads(response)
530 538
531 539 except json.JSONDecodeError as e:
532 540 msg = 'Failed to get a Jy/K factor from DB: JSON Syntax error. {}'.format(e)
533 541 casalog.post(msg)
534 542 raise RuntimeError(msg)
535 543
536 544 def _check_retval(self, retval):
537 - """ Check if 'success' of retval dict is True.
545 + """Check if 'success' of retval dict is True.
538 546
539 547 This method only checks if the api was able to complete the process successfully or not.
540 548 It is expected that 'success' will be False as a response, so the mothod does not raise
541 549 RuntimeError. If the 'success' is False, the *Transelator classes will reject the factor
542 550 value.
543 551 """
544 552 if not retval['success']:
545 553 msg = 'Failed to get a Jy/K factor from DB: {}'.format(retval['error'])
546 554 casalog.post(msg)
547 555
646 654 def _dataset_to_cal_dict(dataset, _extract_factor):
647 655 return_data = []
648 656
649 657 for data in dataset:
650 658 # aux is dictionary holding vis and spw id
651 659 aux = data['aux']
652 660 if not isinstance(aux, dict):
653 661 raise TypeError('The response.aux in the JSON obtained from Jy/K db must be dict.')
654 662
655 663 if 'vis' not in aux:
656 - raise KeyError('The response.aux in the JSON obtained from Jy/K db must contain vis.')
664 + raise KeyError(
665 + 'The response.aux in the JSON obtained from Jy/K db must contain vis.')
657 666
658 667 if 'spwid' not in aux:
659 - raise KeyError('The response.aux in the JSON obtained from Jy/K db must contain spwid.')
668 + raise KeyError(
669 + 'The response.aux in the JSON obtained from Jy/K db must contain spwid.')
660 670
661 671 spwid = aux['spwid']
662 672 if not isinstance(spwid, int):
663 - raise TypeError('The response.aux.spwid in the JSON obtained from Jy/K db must be int.')
673 + raise TypeError(
674 + 'The response.aux.spwid in the JSON obtained from Jy/K db must be int.')
664 675
665 676 vis = aux['vis']
666 677 if not isinstance(vis, str):
667 - raise TypeError('The response.aux.vis in the JSON obtained from Jy/K db must be str.')
678 + raise TypeError(
679 + 'The response.aux.vis in the JSON obtained from Jy/K db must be str.')
668 680
669 681 basename = os.path.basename(os.path.abspath(vis))
670 682
671 683 factor = _extract_factor(data)
672 684 polarization = 'I'
673 685 antenna = data['response']['query']['antenna']
674 686
675 687 return_data.append({'MS': basename, 'Antenna': antenna, 'Spwid': spwid,
676 688 'Polarization': polarization, 'factor': factor})
677 689 return return_data

Everything looks good. We'll let you know here if there's anything you should know about.

Add shortcut