Commits
Sandra M Castro authored ed1f53d4a74 Merge
8 8 | import string |
9 9 | from time import sleep |
10 10 | from urllib.error import HTTPError, URLError |
11 11 | from urllib.parse import urlencode |
12 12 | from urllib.request import urlopen |
13 13 | |
14 14 | import certifi |
15 15 | import numpy as np |
16 16 | |
17 17 | from casatasks import casalog |
18 - | from casatasks.private.sdutil import table_selector, table_manager, tool_manager |
18 + | from casatasks.private.sdutil import (table_manager, table_selector, |
19 + | tool_manager) |
19 20 | from casatools import measures |
20 21 | from casatools import ms as mstool |
21 22 | from casatools import msmetadata, quanta |
22 23 | |
23 24 | |
24 25 | # Jy/K DB part |
25 26 | def gen_factor_via_web_api(vis, spw='*', |
26 27 | endpoint='asdm', |
27 28 | timeout=180, retry=3, retry_wait_time=5): |
28 29 | """Generate factors via Jy/K Web API. |
103 104 | def _vis_to_uid(vis): |
104 105 | """Convert MS name like 'uid___A002_Xabcd_X012 into uid://A002/Xabcd/X012'. |
105 106 | |
106 107 | Arguments: |
107 108 | vis {str} -- The file path of the visibility data. |
108 109 | |
109 110 | Returns: |
110 111 | str -- Corresponding ASDM uid. |
111 112 | """ |
112 113 | basename = os.path.basename(os.path.abspath(vis)) |
113 - | pattern = '^uid___A[0-9][0-9][0-9]_X[0-9a-f]+_X[0-9a-f]+\.ms$' |
114 + | pattern = r'^uid___A[0-9][0-9][0-9]_X[0-9a-f]+_X[0-9a-f]+\.ms$' |
114 115 | if re.match(pattern, basename): |
115 116 | return basename.replace('___', '://').replace('_', '/').replace('.ms', '') |
116 117 | else: |
117 118 | raise RuntimeError('MS name is not appropriate for DB query: {}'.format(basename)) |
118 119 | |
119 120 | |
120 121 | class InterpolationParamsGenerator(): |
121 122 | """A class to generate required parameters for Jy/K Web API with interpolation. |
122 123 | |
123 124 | Usage: |
129 130 | def get_params(cls, vis, spw='*'): |
130 131 | if spw == '': |
131 132 | spw = '*' |
132 133 | |
133 134 | if spw == '*': |
134 135 | spw = cls._get_available_spw(vis, spw) |
135 136 | |
136 137 | params = {} |
137 138 | |
138 139 | science_windows = cls._get_science_windows(vis, spw) |
139 - | timerange, antenna_names, basebands, mean_freqs, spwnames = cls._extract_msmetadata(science_windows, vis) |
140 + | timerange, antenna_names, basebands, mean_freqs, spwnames = \ |
141 + | cls._extract_msmetadata(science_windows, vis) |
140 142 | |
141 143 | mean_freqs = cls._get_mean_freqs(vis, science_windows) |
142 144 | bands = Bands.get(science_windows, spwnames, mean_freqs, vis) |
143 145 | |
144 146 | params['date'] = cls._mjd_to_datestring(timerange['begin']) |
145 147 | params['temperature'] = cls._get_mean_temperature(vis) |
146 148 | params.update(cls._get_aux_params()) |
147 149 | |
148 150 | for antenna_id, antenna_name in enumerate(antenna_names): |
149 151 | params['antenna'] = antenna_name |
173 175 | spwnames = msmd.namesforspws(science_windows) |
174 176 | |
175 177 | return timerange, antenna_names, basebands, mean_freqs, spwnames |
176 178 | |
177 179 | |
178 180 | def _get_available_spw(vis, spw): |
179 181 | science_windows = InterpolationParamsGenerator._get_science_windows(vis, spw=spw) |
180 182 | with tool_manager(vis, msmetadata) as msmd: |
181 183 | spwnames = msmd.namesforspws(science_windows) |
182 184 | |
183 - | spw = ','.join(map(str, [i for i, name in enumerate(spwnames) if not name.startswith('WVR')])) |
185 + | spw = ','.join( |
186 + | map(str, [i for i, name in enumerate(spwnames) |
187 + | if not name.startswith('WVR')])) |
184 188 | return spw |
185 189 | |
186 190 | |
187 191 | def _mjd_to_datestring(epoch): |
188 192 | me = measures() |
189 193 | qa = quanta() |
190 194 | |
191 195 | if epoch['refer'] != 'UTC': |
192 196 | try: |
193 197 | epoch = me.measure(epoch, 'UTC') |
236 240 | def get(cls, science_windows, spwnames, mean_freqs, vis): |
237 241 | """Return all bands corresponding to the 'science_window' given in the input. |
238 242 | |
239 243 | First the method scan 'spwnames', if the band can be detect, the method will |
240 244 | adopt this value. In other case, the method compare the freq with the 'mean_freqs' |
241 245 | at which the band was detect, the method detect the band from the frequencies |
242 246 | that are closest to the result. |
243 247 | """ |
244 248 | bands = cls._extract_bands_from_spwnames(science_windows, spwnames) |
245 249 | mean_freqs_with_undetected_band = cls._filter_mean_freqs_with_undetected_band( |
246 - | science_windows, spwnames, mean_freqs) |
250 + | science_windows, spwnames, mean_freqs) |
247 251 | if len(mean_freqs_with_undetected_band) > 0: |
248 252 | bands.update( |
249 253 | cls._detect_bands_from_mean_freqs(mean_freqs_with_undetected_band, vis) |
250 254 | ) |
251 255 | return bands |
252 256 | |
253 257 | |
254 258 | def _extract_bands_from_spwnames(science_windows, spwnames): |
255 259 | """Extract bands that contain band information in the spwname. |
256 260 | |
355 359 | science_spw = list(np.intersect1d( |
356 360 | msmd.almaspws(tdm=True, fdm=True), |
357 361 | msmd.spwsforintent('OBSERVE_TARGET#ON_SOURCE') |
358 362 | )) |
359 363 | science_dd = [msmd.datadescids(spw=i)[0] for i in science_spw] |
360 364 | |
361 365 | return science_dd |
362 366 | |
363 367 | |
364 368 | def _query_rows(vis, science_dd, stateid, antenna_id): |
365 - | query = f'ANTENNA1=={antenna_id}&&ANTENNA2=={antenna_id}&&DATA_DESC_ID=={science_dd[0]}&&STATE_ID IN {list(stateid)}' |
369 + | query = f'ANTENNA1=={antenna_id}&&ANTENNA2=={antenna_id}&&DATA_DESC_ID=={science_dd[0]}' + \ |
370 + | f'&&STATE_ID IN {list(stateid)}' |
366 371 | with table_selector(vis, query) as tb: |
367 372 | rows = tb.rownumbers() |
368 373 | |
369 374 | return rows |
370 375 | |
371 376 | |
372 377 | def _calc_elevation_mean(rows, vis): |
373 378 | elevations = [] |
374 379 | qa = quanta() |
375 380 | |
400 405 | manager = RequestsManager(client) |
401 406 | manager.get(params) |
402 407 | """ |
403 408 | |
404 409 | def __init__(self, client): |
405 410 | """Set client.""" |
406 411 | self.client = client |
407 412 | |
408 413 | def get(self, params): |
409 414 | """Get the responses of the Jy/K DB.""" |
410 - | dataset = [{'response': self.client.get(param.param), 'aux': param.subparam} for param in params] |
415 + | dataset = [ |
416 + | {'response': self.client.get(param.param), 'aux': param.subparam} |
417 + | for param in params] |
411 418 | return self._filter_success_is_true(dataset) |
412 419 | |
413 420 | def _filter_success_is_true(self, dataset): |
414 421 | return [data for data in dataset if data['response']['success']] |
415 422 | |
416 423 | |
417 424 | class JyPerKDatabaseClient(): |
418 425 | """A class to get values from Jy/K Web API. |
419 426 | |
420 427 | The Jy/K Web API address is 'https://asa.alma.cl/science/jy-kelvins'. The address |
430 437 | endpoint {str} -- The endpoint of Jy/K DB Web API to access. Options are |
431 438 | 'asdm' (default), 'model-fit', 'interpolation'. |
432 439 | timeout {int} --- Maximum waiting time [sec] for the Web API access, defaults |
433 440 | to 180 sec. |
434 441 | retry {int} -- Number of retry when the Web API access fails, defaults to 3 |
435 442 | times. |
436 443 | retry_wait_time {int} -- Waiting time [sec] until next query when the Web API |
437 444 | access fails, defaults to 5 sec. |
438 445 | """ |
439 446 | assert endpoint in ['asdm', 'model-fit', 'interpolation'], \ |
440 - | 'The JyPerKDatabaseClient class requires one of endpoint: asdm, model-fit or interpolation' |
447 + | 'The JyPerKDatabaseClient class requires one of endpoint: ' \ |
448 + | 'asdm, model-fit or interpolation' |
441 449 | self.web_api_url = self._generate_web_api_url(endpoint) |
442 450 | self.timeout = timeout |
443 451 | self.retry = retry |
444 452 | self.retry_wait_time = retry_wait_time |
445 453 | |
446 454 | def get(self, param): |
447 455 | """Get the Web API response. |
448 456 | |
449 457 | Arguments: |
450 458 | param {dict} -- The parameters used in the Web API. |
499 507 | msg = 'Failed to load URL: {0}\n'.format(url) \ |
500 508 | + 'Error Message: URLError(Reason="{0}")\n'.format(e.reason) |
501 509 | casalog.post(msg) |
502 510 | return {'status': 'URLError', 'err_msg': msg} |
503 511 | except socket_timeout as e: # not connect |
504 512 | msg = 'Failed to load URL: {0}\n'.format(url) \ |
505 513 | + 'Error Message: URLError(Reason="{0}")\n'.format(e) |
506 514 | casalog.post(msg) |
507 515 | return {'status': 'URLError', 'err_msg': msg} |
508 516 | |
509 - | |
510 517 | def _try_to_get_response(self, url): |
511 518 | casalog.post(f'Accessing Jy/K DB: request URL is "{url}"') |
512 519 | for i in range(self.retry): |
513 520 | response_with_tag = self._retrieve(url) |
514 521 | if response_with_tag['status'] == 'Success': |
515 - | casalog.post(f'Got a response successfully') |
522 + | casalog.post('Got a response successfully') |
516 523 | return response_with_tag['body'] |
517 - | |
524 + | |
518 525 | if i < self.retry - 1: |
519 526 | casalog.post(response_with_tag['err_msg']) |
520 - | casalog.post(f'Sleeping for {str(self.retry_wait_time)} seconds because the request failed') |
527 + | casalog.post( |
528 + | f'Sleeping for {str(self.retry_wait_time)} seconds because the request failed') |
521 529 | sleep(self.retry_wait_time) |
522 530 | casalog.post(f'Retry to access Jy/K DB ({str(i + 2)}/{str(self.retry)})') |
523 531 | |
524 532 | if response_with_tag['status'] != 'Success': |
525 533 | raise RuntimeError(response_with_tag['err_msg']) |
526 534 | |
527 535 | def _convert_to_json(self, response): |
528 536 | try: |
529 537 | return json.loads(response) |
530 538 | |
531 539 | except json.JSONDecodeError as e: |
532 540 | msg = 'Failed to get a Jy/K factor from DB: JSON Syntax error. {}'.format(e) |
533 541 | casalog.post(msg) |
534 542 | raise RuntimeError(msg) |
535 543 | |
536 544 | def _check_retval(self, retval): |
537 - | """ Check if 'success' of retval dict is True. |
545 + | """Check if 'success' of retval dict is True. |
538 546 | |
539 547 | This method only checks if the api was able to complete the process successfully or not. |
540 548 | It is expected that 'success' will be False as a response, so the mothod does not raise |
541 549 | RuntimeError. If the 'success' is False, the *Transelator classes will reject the factor |
542 550 | value. |
543 551 | """ |
544 552 | if not retval['success']: |
545 553 | msg = 'Failed to get a Jy/K factor from DB: {}'.format(retval['error']) |
546 554 | casalog.post(msg) |
547 555 | |
646 654 | def _dataset_to_cal_dict(dataset, _extract_factor): |
647 655 | return_data = [] |
648 656 | |
649 657 | for data in dataset: |
650 658 | # aux is dictionary holding vis and spw id |
651 659 | aux = data['aux'] |
652 660 | if not isinstance(aux, dict): |
653 661 | raise TypeError('The response.aux in the JSON obtained from Jy/K db must be dict.') |
654 662 | |
655 663 | if 'vis' not in aux: |
656 - | raise KeyError('The response.aux in the JSON obtained from Jy/K db must contain vis.') |
664 + | raise KeyError( |
665 + | 'The response.aux in the JSON obtained from Jy/K db must contain vis.') |
657 666 | |
658 667 | if 'spwid' not in aux: |
659 - | raise KeyError('The response.aux in the JSON obtained from Jy/K db must contain spwid.') |
668 + | raise KeyError( |
669 + | 'The response.aux in the JSON obtained from Jy/K db must contain spwid.') |
660 670 | |
661 671 | spwid = aux['spwid'] |
662 672 | if not isinstance(spwid, int): |
663 - | raise TypeError('The response.aux.spwid in the JSON obtained from Jy/K db must be int.') |
673 + | raise TypeError( |
674 + | 'The response.aux.spwid in the JSON obtained from Jy/K db must be int.') |
664 675 | |
665 676 | vis = aux['vis'] |
666 677 | if not isinstance(vis, str): |
667 - | raise TypeError('The response.aux.vis in the JSON obtained from Jy/K db must be str.') |
678 + | raise TypeError( |
679 + | 'The response.aux.vis in the JSON obtained from Jy/K db must be str.') |
668 680 | |
669 681 | basename = os.path.basename(os.path.abspath(vis)) |
670 682 | |
671 683 | factor = _extract_factor(data) |
672 684 | polarization = 'I' |
673 685 | antenna = data['response']['query']['antenna'] |
674 686 | |
675 687 | return_data.append({'MS': basename, 'Antenna': antenna, 'Spwid': spwid, |
676 688 | 'Polarization': polarization, 'factor': factor}) |
677 689 | return return_data |