Commits

gcwrap/python/scripts/regressions/admin/casaTestHelper.py

Added+x
1 +"""
2 +casa6tools = [
3 + "agentflagger", "atcafiller", "atmosphere", "calanalysis", "calibrater", "coercetype", "componentlist", "config", "constants", "coordsys", "ctuser", "functional", "image",
4 + "imagemetadata", "imagepol", "imager", "iterbotsink", "logsink", "measures", "miriadfiller", "ms", "msmetadata", "mstransformer", "platform", "quanta", "regionmanager", "sakura",
5 + "sdm", "simulator", "singledishms", "spectralline", "synthesisdeconvolver", "synthesisimager", "synthesisimstore", "synthesisnormalizer", "synthesisutils", "table", "typecheck", "utils",
6 + "vlafiller", "vpmanager"
7 + ]
8 +"""
9 +casa6tasks = set([
10 + "accor", "accum", "applycal", "asdmsummary", "bandpass", "blcal", "calstat","clearcal", "clearstat", "concat", "conjugatevis", "cvel", "cvel2",
11 + "delmod", "exportasdm", "exportfits", "exportuvfits", "feather", "fixplanets","fixvis", "flagcmd", "flagdata", "flagmanager", "fluxscale", "ft", "gaincal",
12 + "gencal", "hanningsmooth", "imcollapse", "imcontsub", "imdev", "imfit", "imhead","imhistory", "immath", "immoments", "impbcor", "importasap", "importasdm",
13 + "importatca", "importfits", "importfitsidi", "importgmrt", "importmiriad","importnro", "importuvfits", "importvla", "impv", "imrebin", "imreframe",
14 + "imregrid", "imsmooth", "imstat", "imsubimage", "imtrans", "imval","initweights", "listcal", "listfits", "listhistory", "listobs", "listpartition",
15 + "listsdm", "listvis", "makemask", "mstransform", "partition", "polcal","predictcomp", "rerefant", "rmfit", "rmtables", "sdbaseline", "sdcal",
16 + "sdfit", "sdfixscan", "sdgaincal", "sdimaging", "sdsmooth", "setjy","simalma", "simanalyze", "simobserve", "slsearch", "smoothcal", "specfit",
17 + "specflux", "specsmooth", "splattotable", "split", "spxfit", "statwt","tclean", "uvcontsub", "uvmodelfit", "uvsub", "virtualconcat", "vishead", "visstat","widebandpbcor" ])
18 +
19 +miscellaneous_tasks = set(['wvrgcal','plotms'])
20 +
21 +import os, sys, time
22 +from functools import wraps
23 +
24 +import fnmatch
25 +import logging
26 +import filecmp
27 +import unittest
28 +import pickle
29 +import numpy
30 +import math
31 +import numbers
32 +import six
33 +import operator
34 +import subprocess
35 +
36 +logging.basicConfig(level=logging.INFO,format='%(message)s')
37 +#logging.basicConfig(level=logging.DEBUG,format='%(levelname)s-%(message)s')
38 +
39 +"""
40 +
41 +logging.debug('This is a debug message')
42 +logging.info('This is an info message')
43 +logging.warning('This is a warning message')
44 +logging.error('This is an error message')
45 +logging.critical('This is a critical message')
46 +"""
47 +casa5 = False
48 +casa6 = False
49 +
50 +try:
51 + # CASA 6
52 + logging.debug("Importing CASAtools")
53 + import casatools
54 + logging.debug("Importing CASAtasks")
55 + import casatasks
56 +
57 + tb = casatools.table()
58 + tb2 = casatools.table()
59 + tbt = casatools.table()
60 + ms = casatools.ms()
61 + ia = casatools.image()
62 +
63 + from casatasks import casalog
64 + casa6 = True
65 +except ImportError:
66 + # CASA 5
67 + logging.debug("Import casa6 errors. Trying CASA5...")
68 + from __main__ import default
69 + from taskinit import tbtool, mstool, iatool
70 + from taskinit import *
71 + from casa_stack_manip import stack_find, find_casa
72 +
73 + tb = tbtool()
74 + tb2 = tbtool()
75 + tbt = tbtool()
76 + ms = mstool()
77 + ia = iatool()
78 +
79 + casa = find_casa( )
80 + if casa.has_key('state') and casa['state'].has_key('init_version') and casa['state']['init_version'] > 0:
81 + casaglobals=True
82 + casac = stack_find("casac")
83 + casalog = stack_find("casalog")
84 +
85 + casa5 = True
86 +
87 +############################################################################################
88 +################################## Classes ##################################
89 +############################################################################################
90 +
91 +# Logger
92 +class Logger:
93 + #TODO: This class needs work
94 + import sys
95 + import logging
96 +
97 + def verbose_logging_start():
98 + logger = logging.getLogger()
99 + logger.level = logging.DEBUG
100 + stream_handler = logging.StreamHandler(sys.stdout)
101 + logger.addHandler(stream_handler)
102 +
103 + def verbose_logging_stop():
104 + pass
105 +
106 +
107 +# Weblog
108 +class Weblog:
109 + def __init__(self, taskname, localdict):
110 + self.localdict = localdict
111 + self.taskname = taskname
112 +
113 + def write_modal_style(self):
114 +
115 +
116 + html.write(' /* Style the Image Used to Trigger the Modal */' + '\n')
117 + html.write('.myImg {' + '\n')
118 + html.write('border-radius: 5px; cursor: pointer; transition: 0.3s; }'+ '\n')
119 +
120 + html.write('.myImg:hover{' + '\n')
121 + html.write('opacity: 0.7;}'+ '\n')
122 +
123 + html.write('/* The Modal (background) */' + '\n')
124 + html.write('.modal {' + '\n')
125 + html.write(' display: none; /* Hidden by default */' + '\n')
126 + html.write(' position: fixed; /* Stay in place */' + '\n')
127 + html.write(' z-index: 1; /* Sit on top */' + '\n')
128 + html.write(' padding-top: 100px; /* Location of the box */' + '\n')
129 + html.write(' left: 0;' + '\n')
130 + html.write(' top: 0;' + '\n')
131 + html.write(' width: 100%; /* Full width */' + '\n')
132 + html.write(' height: 100%; /* Full height */' + '\n')
133 + html.write(' overflow: auto; /* Enable scroll if needed */' + '\n')
134 + html.write(' background-color: rgb(0,0,0); /* Fallback color */' + '\n')
135 + html.write(' background-color: rgba(0,0,0,0.9); /* Black w/ opacity */' + '\n')
136 + html.write('}' + '\n')
137 + html.write('/* Modal Content (Image) */' + '\n')
138 + html.write('.modal-content {' + '\n')
139 + html.write(' margin: auto;' + '\n')
140 + html.write(' display: block;' + '\n')
141 + html.write(' width: 80%;' + '\n')
142 + html.write(' max-width: 700px;' + '\n')
143 + html.write('}' + '\n')
144 +
145 + html.write('/* Caption of Modal Image (Image Text) - Same Width as the Image */' + '\n')
146 + html.write('#caption {' + '\n')
147 + html.write(' margin: auto;' + '\n')
148 + html.write(' display: block;' + '\n')
149 + html.write(' width: 80%;' + '\n')
150 + html.write(' max-width: 700px;' + '\n')
151 + html.write(' text-align: center;' + '\n')
152 + html.write(' color: #ccc;' + '\n')
153 + html.write(' padding: 10px 0;' + '\n')
154 + html.write(' height: 150px;' + '\n')
155 + html.write('}' + '\n')
156 +
157 + html.write('/* Add Animation - Zoom in the Modal */' + '\n')
158 + html.write('.modal-content, #caption {' + '\n')
159 + html.write(' animation-name: zoom;' + '\n')
160 + html.write(' animation-duration: 0.6s;' + '\n')
161 + html.write('}' + '\n')
162 +
163 + html.write('@keyframes zoom {' + '\n')
164 + html.write(' from {transform:scale(0)}' + '\n')
165 + html.write(' to {transform:scale(1)}' + '\n')
166 + html.write('}' + '\n')
167 +
168 + html.write('/* The Close Button */' + '\n')
169 + html.write('.close {' + '\n')
170 + html.write(' position: absolute;' + '\n')
171 + html.write(' top: 15px;' + '\n')
172 + html.write(' right: 35px;' + '\n')
173 + html.write(' color: #f1f1f1;' + '\n')
174 + html.write(' font-size: 40px;' + '\n')
175 + html.write(' font-weight: bold;' + '\n')
176 + html.write(' transition: 0.3s;' + '\n')
177 + html.write('}' + '\n')
178 +
179 + html.write('.close:hover,' + '\n')
180 + html.write('.close:focus {' + '\n')
181 + html.write(' color: #bbb;' + '\n')
182 + html.write(' text-decoration: none;' + '\n')
183 + html.write(' cursor: pointer;' + '\n')
184 + html.write('}' + '\n')
185 +
186 + html.write('/* 100% Image Width on Smaller Screens */' + '\n')
187 + html.write('@media only screen and (max-width: 700px){' + '\n')
188 + html.write(' .modal-content {' + '\n')
189 + html.write(' width: 100%;' + '\n')
190 + html.write(' }' + '\n')
191 + html.write('} ' + '\n')
192 +
193 + def generate_header(self, testname):
194 + html.write('<!doctype html>' + '\n')
195 + html.write('<html lang="en">' + '\n')
196 + html.write('<head>' + '\n')
197 + html.write('<meta charset="utf-8">' + '\n')
198 + html.write('<title>{}</title>'.format(testname) + '\n')
199 + html.write('<meta name="description" content="The HTML5 Herald">' + '\n')
200 + html.write('<meta name="author" content="SitePoint">' + '\n')
201 + html.write('<link rel="stylesheet" href="css/styles.css?v=1.0">' + '\n')
202 + html.write('</head>' + '\n')
203 + html.write('<body>' + '\n')
204 + html.write('<script src="js/scripts.js"></script>' + '\n')
205 + html.write('<h1>{}</h1>'.format(testname) + '\n')
206 +
207 +
208 + def generate_status_table_style(self,dictionary):
209 + html.write('<style type="text/css">' + '\n')
210 + html.write('.collapsible {background-color: #777;color: white;cursor: pointer;padding: 18px;width: 100%;border: none;text-align: left;outline: none;font-size: 15px;}' + '\n')
211 + html.write('.active, .collapsible:hover { background-color: #555;}' + '\n')
212 + html.write('.content {padding: 0 18px;display: none;overflow: hidden;background-color: #f1f1f1;}' + '\n')
213 + html.write(".boxed { border: 1px solid green ;padding: 0 5px 0 5px;margin: 50px}" + '\n')
214 + html.write('.tg {border-collapse:collapse;border-spacing:0;}' + '\n')
215 + html.write('.tg td{font-family:Arial, sans-serif;font-size:14px;padding:10px 5px;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:black;}' + '\n')
216 + html.write('.tg th{font-family:Arial, sans-serif;font-size:14px;font-weight:normal;padding:10px 5px;border-style:solid;border-width:1px;overflow:hidden;word-break:normal;border-color:black;}' + '\n')
217 + html.write('.tg .tg-0lax{text-align:left;vertical-align:top}' + '\n')
218 + html.write('.tg .tg-ck9b{background-color:#009901;color:#32cb00;text-align:left;vertical-align:top}' + '\n')
219 + html.write('.tg .tg-r50r{background-color:#cb0000;text-align:left;vertical-align:top}' + '\n')
220 + html.write('.tg-sort-header::-moz-selection{background:0 0}.tg-sort-header::selection{background:0 0}.tg-sort-header{cursor:pointer}.tg-sort-header:after{content:'';float:right;margin-top:7px;border-width:0 5px 5px;border-style:solid;border-color:#404040 transparent;visibility:hidden}.tg-sort-header:hover:after{visibility:visible}.tg-sort-asc:after,.tg-sort-asc:hover:after,.tg-sort-desc:after{visibility:visible;opacity:.4}.tg-sort-desc:after{border-bottom:none;border-width:5px 5px 0}' + '\n')
221 +
222 + Weblog(self.taskname, self.localdict).write_modal_style()
223 +
224 + html.write('</style>' + '\n')
225 +
226 + def generate_tail(self,dictionary):
227 + html.write('<script>' + '\n')
228 + html.write('var coll = document.getElementsByClassName("collapsible");' + '\n')
229 + html.write('var i;' + '\n')
230 + html.write('for (i = 0; i < coll.length; i++) {' + '\n')
231 + html.write(' coll[i].addEventListener("click", function() {' + '\n')
232 + html.write(' this.classList.toggle("active");' + '\n')
233 + html.write(' var content = this.nextElementSibling;' + '\n')
234 + html.write(' if (content.style.display === "block") {' + '\n')
235 + html.write(' content.style.display = "none";' + '\n')
236 + html.write(' } else {' + '\n')
237 + html.write(' content.style.display = "block";' + '\n')
238 + html.write(' }' + '\n')
239 + html.write(' });' + '\n')
240 + html.write('}' + '\n')
241 +
242 + html.write('// Get the modal' + '\n')
243 + html.write("var modal = document.getElementById('myModal');" + '\n')
244 + html.write('// Get the image and insert it inside the modal - use its "alt" text as a caption' + '\n')
245 + html.write("var img = $('.myImg');" + '\n')
246 + html.write('var modalImg = $("#img01");' + '\n')
247 + html.write('var captionText = document.getElementById("caption");' + '\n')
248 + html.write("$('.myImg').click(function(){" + '\n')
249 + html.write(' modal.style.display = "block";' + '\n')
250 + html.write(' var newSrc = this.src;' + '\n')
251 + html.write(" modalImg.attr('src', newSrc);" + '\n')
252 + html.write(' captionText.innerHTML = this.alt;' + '\n')
253 + html.write('});' + '\n')
254 + html.write('// Get the <span> element that closes the modal' + '\n')
255 + html.write('var span = document.getElementsByClassName("close")[0];' + '\n')
256 + html.write('// When the user clicks on <span> (x), close the modal' + '\n')
257 + html.write('span.onclick = function() {' + '\n')
258 + html.write(' modal.style.display = "none";' + '\n')
259 + html.write('}' + '\n')
260 +
261 + html.write('</script>' + '\n')
262 + html.write("</body>" + '\n')
263 + html.write("</html>" + '\n')
264 +
265 + def generate_table_row(self, test, description, runtime, status_color):
266 +
267 + html.write("<tr>" + "\n")
268 + html.write('<td class="tg-0lax">{}</td>'.format(test) + '\n')
269 + html.write('<td class="tg-0lax">{}</td>'.format(description) + '\n')
270 + html.write('<td class="tg-0lax">{}s</td>'.format(round(runtime,2)) + '\n')
271 + html.write('<td class={}></td>'.format(status_color) + '\n')
272 + html.write("</tr>" + "\n")
273 +
274 + def generate_status_table(self, dictionary):
275 + html.write('<table id="tg-cC48w" class="tg">' + '\n')
276 + html.write('<tr>' + '\n')
277 + html.write('<th class="tg-0lax">Test Name</th>' + '\n')
278 + html.write('<th class="tg-0lax">Description </th>' + '\n')
279 + html.write('<th class="tg-0lax">Run Time</th>' + '\n')
280 + html.write('<th class="tg-0lax">Status</th>' + '\n')
281 + html.write('</tr>' + '\n')
282 +
283 + for key, value in dictionary.items():
284 + Weblog(self.taskname, self.localdict).generate_table_row(str(key), dictionary[key]['description'], dictionary[key]['runtime'], "tg-ck9b" if dictionary[key]['status'] == True else "tg-r50r" )
285 + html.write('</table>' + '\n')
286 +
287 + def generate_summary_box(self, dictionary):
288 + for key, value in dictionary.items():
289 + html.write('<button class="collapsible">{}</button>'.format(key) + '\n')
290 + html.write('<div class="content">'+ '\n')
291 + html.write('<div class="boxed">'+ '\n')
292 + html.write('<h3>{}</h3>'.format(key) + '\n')
293 + html.write('<i><sub>{}</sub></i>'.format(dictionary[key]['description'])+ '\n')
294 + html.write('<p><b>Elapsed Time:</b> {} Seconds</p>'.format(dictionary[key]['runtime'])+ '\n')
295 + html.write('<p><b>Status:</b> {}</p>'.format("PASS" if dictionary[key]['status'] == True else "FAIL" )+ '\n')
296 + html.write('<p><b>Task Executions:</b></p>'+ '\n')
297 + Weblog(self.taskname, self.localdict).write_inline_list( dictionary[key]['taskcall'] )
298 + Weblog(self.taskname, self.localdict).add_miscellaneous_info(dictionary[key])
299 + html.write('<p><b>Re-Run:</b> {}</p>'.format(dictionary[key]['rerun'])+ '\n')
300 + html.write('</div>' + '\n')
301 + html.write('</div>' + '\n')
302 +
303 + def write_inline_list(self, array):
304 + html.write('<ul>' + '\n')
305 + for item in array:
306 + html.write('<li>{}</li>'.format(item) + '\n')
307 + if str(item).endswith(".png"):
308 + html.write('<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>'+ '\n')
309 + html.write('<img class="myImg" src="{}" alt="{}" height="300" width="300">'.format(item, item) + '\n')
310 + html.write('<div id="myModal" class="modal">'+ '\n')
311 + html.write(' <span class="close" onclick="document.getElementById(\'myModal\').style.display=\'none\'">&times;</span>' + '\n')
312 + html.write(' <img class="modal-content" id="img01">' + '\n')
313 + html.write(' <div id="caption"></div>' + '\n')
314 + html.write('</div>' + '\n')
315 + html.write('</img>' + '\n')
316 + html.write('</ul>' + '\n')
317 +
318 + def write_inline_dict(self, dictionary):
319 + html.write('<ul>' + '\n')
320 + for key, value in sorted(dictionary.items()):
321 + html.write('<p><b>{}</b>: {}</p>'.format(key,value)+ '\n')
322 + html.write('</ul>' + '\n')
323 +
324 + def add_miscellaneous_info(self, subdictionary):
325 + default_keys = ['description','status','runtime','taskcall','rerun']
326 + for key, value in subdictionary.items():
327 + if key in default_keys:
328 + continue
329 + if type(subdictionary[key]) == list:
330 + html.write('<p><b>{}:</b></p>'.format(key)+ '\n')
331 + Weblog(self.taskname, self.localdict).write_inline_list(subdictionary[key])
332 + elif type(subdictionary[key]) == dict:
333 + html.write('<p><b>{}:</b></p>'.format(key)+ '\n')
334 + Weblog(self.taskname, self.localdict).write_inline_dict(subdictionary[key])
335 + else:
336 + html.write('<span style="white-space: pre-line"><p><b>{}:</b> {}</p></span>'.format(key,subdictionary[key])+ '\n')
337 +
338 + def generate_weblog(self):
339 + logging.debug("Generating Weblog: {}".format(self.taskname))
340 + Weblog(self.taskname, self.localdict).generate_header("Test {}".format(self.taskname))
341 + Weblog(self.taskname, self.localdict).generate_status_table_style(self.localdict)
342 + Weblog(self.taskname, self.localdict).generate_status_table(self.localdict)
343 + Weblog(self.taskname, self.localdict).generate_summary_box(self.localdict)
344 + Weblog(self.taskname, self.localdict).generate_tail(self.localdict)
345 +
346 +############################################################################################
347 +################################## Functions ##################################
348 +############################################################################################
349 +def compare_CASA_variable_cols(referencetab, testtab, varcol, tolerance=0.0):
350 + '''
351 + compare_CASA_variable_cols - Compare a variable column of two CASA tables.
352 + @param referencetab --> a reference table
353 + @param testtab --> a table to verify
354 + @param varcol --> the name of a variable column (str)
355 + @param tolerance --> Tolerance
356 +
357 + @return: True if reference tab == test table else False
358 + '''
359 + logging.info("Comparing Column: {} within {} and {}".format(varcol,referencetab, testtab))
360 + logging.debug("Executing: compare_CASA_variable_cols(referencetab={},testtab={}, varcol={}, tolerance={})".format(referencetab, testtab, varcol, tolerance))
361 + retval = True
362 +
363 + tb.open(referencetab)
364 + cnames = tb.colnames()
365 +
366 + tb2.open(testtab)
367 + col = varcol
368 + if tb.isvarcol(col) and tb2.isvarcol(col):
369 + try:
370 + # First check
371 + if tb.nrows() != tb2.nrows():
372 + logging.error('Length of {} differ from {}, {} != {}'.format(referencetab,testtab,tb.nrows(),tb2.nrows()))
373 + retval = False
374 + else:
375 + for therow in range(tb.nrows()):
376 + rdata = tb.getcell(col,therow)
377 + tdata = tb2.getcell(col,therow)
378 +
379 + if not rdata.all()==tdata.all():
380 + if (tolerance>0.0):
381 + differs=False
382 + for j in range(0,len(rdata)):
383 +
384 + if ((isinstance(rdata[j],float)) or (isinstance(rdata[j],int))):
385 + if (abs(rdata[j]-tdata[j]) > tolerance*abs(rdata[j]+tdata[j])):
386 +# print('Column ', col,' differs in tables ', referencetab, ' and ', testtab)
387 +# print(therow, j)
388 +# print(rdata[j])
389 +# print(tdata[j])
390 + differs = True
391 + elif (isinstance(rdata[j],list)) or (isinstance(rdata[j],numpy.ndarray)):
392 + for k in range(0,len(rdata[j])):
393 + if (abs(rdata[j][k]-tdata[j][k]) > tolerance*abs(rdata[j][k]+tdata[j][k])):
394 +# print('Column ', col,' differs in tables ', referencetab, ' and ', testtab)
395 +# print(therow, j, k)
396 +# print(rdata[j][k])
397 +# print(tdata[j][k])
398 + differs = True
399 + if differs:
400 + print('ERROR: Column {} of {} and {} do not agree within tolerance {}'.format(col,referencetab, testtab, tolerance))
401 + retval = False
402 + break
403 + else:
404 + print('ERROR: Column {} of {} and {} do not agree.'.format(col,referencetab, testtab))
405 + print('ERROR: First row to differ is row={}'.format(therow))
406 + retval = False
407 + break
408 + finally:
409 + tb.close()
410 + tb2.close()
411 +
412 + else:
413 + logging.info('Columns are not varcolumns.')
414 + retval = False
415 +
416 + if retval:
417 + logging.info('Column {} of {} and {} agree'.format(col,referencetab, testtab))
418 +
419 + return retval
420 +
421 +
422 +def compare_CASA_tables(referencetab, testtab, excludecols = [], tolerance=0.001, mode="percentage", startrow = 0, nrow = -1, rowincr = 1):
423 + '''
424 + compare_CASA_tables - compare two CASA tables
425 + @param referencetab - the table which is assumed to be correct
426 + @param testtab - the table which is to be compared to referencetab
427 + @param excludecols - list of column names which are to be ignored
428 + @param tolerance - permitted fractional difference (default 0.001 = 0.1 percent)
429 + @param mode - comparison is made as "percentage", "absolute", "phaseabsdeg" (for complex numbers = difference of the phases in degrees)
430 +
431 + @return: True if reference tab == test table else False
432 + '''
433 + logging.info("Comparing {} to {}".format(referencetab, testtab))
434 + logging.debug("Executing: compare_CASA_tables(referencetab = {}, testtab = {}, excludecols = {}, tolerance={}, mode={}, startrow = {}, nrow = {}, rowincr = {})".format(referencetab, testtab, excludecols, tolerance, mode, startrow, nrow , rowincr))
435 +
436 + if not isinstance(excludecols, list):
437 + logging.error("excludecols not in correct format")
438 + raise TypeError("excludecols must be a list")
439 +
440 + if referencetab.endswith("cal") or testtab.endswith("cal"):
441 + logging.warning("WARNING: Will compare caltables using compare_caltables")
442 + return compare_caltables(referencetab, testtab, cols= excludecols, rtol=8e-7, atol=1e-8)
443 +
444 + ##### Begin: Tempory Fix
445 + if len(excludecols) == 0:
446 + excludecols = ["FLAG_CATEGORY"]
447 + else:
448 + excludecols.append('FLAG_CATEGORY')
449 + """
450 + #TODO: Fix Error in checking FLAG_CATEGORY
451 + tb.getcol("FLAG_CATEGORY")
452 + SEVERE getcol::FLAG_CATEGORY Exception Reported: Table DataManager error: Invalid operation: TSM: no array in row 0 of column FLAG_CATEGORY in **
453 + RuntimeError: Table DataManager error: Invalid operation: TSM: no array in row 0 of column FLAG_CATEGORY in **
454 + """
455 + ##### End: Tempory Fix
456 +
457 +
458 + rval = True
459 +
460 + # Open reference table
461 + tb.open(referencetab)
462 + cnames = tb.colnames()
463 +
464 + # Open test table
465 + tb2.open(testtab)
466 + cnames2 = tb2.colnames()
467 +
468 +
469 + if sorted(cnames) != sorted(cnames2):
470 + logging.debug("Available columns in Reference Table {}: {}".format(referencetab,cnames))
471 + logging.debug("Available columns in Test Table{}: {}".format(testtab,cnames2))
472 + return False
473 +
474 + for excludecol in excludecols:
475 + if (excludecol not in cnames) and (excludecol not in cnames2):
476 + logging.warning("Column {} Not in {} or {}. Will Continue without Checking against this column".format(excludecol,referencetab,testtab))
477 + logging.debug("Available columns in Reference Table {}: {}".format(referencetab,cnames))
478 + logging.debug("Available columns in Test Table{}: {}".format(testtab,cnames2))
479 +
480 + try:
481 + for cname in cnames:
482 + if cname in excludecols:
483 + continue
484 +
485 + logging.info("\nTesting column: {}".format(cname))
486 +
487 + a = 0
488 + try:
489 + a = tb.getcol(cname,startrow=startrow,nrow=nrow,rowincr=rowincr)
490 + except:
491 + tb.getcol(cname)
492 + rval = False
493 + logging.critical('Error accessing column ', cname, ' in table ', referencetab)
494 + logging.critical(sys.exc_info()[0])
495 + break
496 +
497 + b = 0
498 + try:
499 + b = tb2.getcol(cname,startrow=startrow,nrow=nrow,rowincr=rowincr)
500 + except:
501 + rval = False
502 + logging.critical('Error accessing column ', cname, ' in table ', testtab)
503 + logging.critical(sys.exc_info()[0])
504 + break
505 +
506 + if not (len(a)==len(b)):
507 + logging.error('Column {} has different length in tables {} and {}'.format(cname, referencetab, testtab))
508 + logging.error(a)
509 + logging.error(b)
510 + rval = False
511 + break
512 + else:
513 + differs = False
514 + if not (a==b).all():
515 + for i in range(0,len(a)):
516 + if (isinstance(a[i],float)):
517 + if ((mode=="percentage") and (abs(a[i]-b[i]) > tolerance*abs(a[i]))) or ((mode=="absolute") and (abs(a[i]-b[i]) > tolerance)):
518 + print("Column " + cname + " differs")
519 + print("Row=" + str(i))
520 + print("Reference file value: " + str(a[i]))
521 + print("Input file value: " + str(b[i]))
522 + if (mode=="percentage"):
523 + print("Tolerance is {0}%; observed difference was {1} %".format (tolerance * 100, 100*abs(a[i]-b[i])/abs(a[i])))
524 + else:
525 + print("Absolute tolerance is {0}; observed difference: {1}".format (tolerance, (abs(a[i]-b[i]))))
526 + differs = True
527 + rval = False
528 + break
529 + elif (isinstance(a[i],int) or isinstance(a[i],numpy.int32)):
530 + if (abs(a[i]-b[i]) > 0):
531 + print("Column " + cname + " differs")
532 + print("Row=" + str(i))
533 + print("Reference file value: " + str(a[i]))
534 + print("Input file value: " + str(b[i]))
535 + if (mode=="percentage"):
536 + print("tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i])))
537 + else:
538 + print("absolute tolerance should be " + str(abs(a[i]-b[i])))
539 + differs = True
540 + rval = False
541 + break
542 + elif (isinstance(a[i],str) or isinstance(a[i],numpy.bool_)):
543 + if not (a[i]==b[i]):
544 + print("Column " + c + " differs")
545 + print("Row=" + str(i))
546 + print("Reference file value: " + str(a[i]))
547 + print("Input file value: " + str(b[i]))
548 + if (mode=="percentage"):
549 + print("tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i])))
550 + else:
551 + print("absolute tolerance should be " + str(abs(a[i]-b[i])))
552 + differs = True
553 + rval = False
554 + break
555 + elif (isinstance(a[i],list)) or (isinstance(a[i],numpy.ndarray)):
556 + for j in range(0,len(a[i])):
557 + if differs: break
558 + if ((isinstance(a[i][j],float)) or (isinstance(a[i][j],int))):
559 + if ((mode=="percentage") and (abs(a[i][j]-b[i][j]) > tolerance*abs(a[i][j]))) or ((mode=="absolute") and (abs(a[i][j]-b[i][j]) > tolerance)):
560 + print("Column " + c + " differs")
561 + print("(Row,Element)=(" + str(j) + "," + str(i) + ")")
562 + print("Reference file value: " + str(a[i][j]))
563 + print("Input file value: " + str(b[i][j]))
564 + if (mode=="percentage"):
565 + print("Tolerance in % should be " + str(100*abs(a[i][j]-b[i][j])/abs(a[i][j])))
566 + else:
567 + print("Absolute tolerance should be " + str(abs(a[i][j]-b[i][j])))
568 + differs = True
569 + rval = False
570 + break
571 + elif (isinstance(a[i][j],list)) or (isinstance(a[i][j],numpy.ndarray)):
572 + it = range(0,len(a[i][j]))
573 + if mode=="percentage":
574 + diff = numpy.abs(numpy.subtract(a[i][j], b[i][j])) > tolerance * numpy.abs(a[i][j])
575 + it = numpy.where(diff)[0]
576 + elif (mode=="absolute"):
577 + diff = numpy.abs(numpy.subtract(a[i][j], b[i][j])) > tolerance
578 + it = numpy.where(diff)[0]
579 + for k in it:
580 + if differs: break
581 + if ( ((mode=="percentage") and (abs(a[i][j][k]-b[i][j][k]) > tolerance*abs(a[i][j][k]))) \
582 + or ((mode=="absolute") and (abs(a[i][j][k]-b[i][j][k]) > tolerance)) \
583 + or ((mode=="phaseabsdeg") and (phasediffabsdeg(a[i][j][k],b[i][j][k])>tolerance)) \
584 + ):
585 + print("Column " + c + " differs")
586 + print("(Row,Channel,Corr)=(" + str(k) + "," + str(j) + "," + str(i) + ")")
587 + print("Reference file value: " + str(a[i][j][k]))
588 + print("Input file value: " + str(b[i][j][k]))
589 + if (mode=="percentage"):
590 + print("Tolerance in % should be " + str(100*abs(a[i][j][k]-b[i][j][k])/abs(a[i][j][k])))
591 + elif (mode=="absolute"):
592 + print("Absolute tolerance should be " + str(abs(a[i][j][k]-b[i][j][k])))
593 + elif (mode=="phaseabsdeg"):
594 + print("Phase tolerance in degrees should be " + str(phasediffabsdeg(a[i][j][k],b[i][j][k])))
595 + else:
596 + print("Unknown comparison mode: ",mode)
597 + differs = True
598 + rval = False
599 + break
600 +
601 + else:
602 + print("Unknown data type: ",type(a[i]))
603 + differs = True
604 + rval = False
605 + break
606 +
607 + if not differs: print("Column " + cname + " PASSED")
608 + finally:
609 + tb.close()
610 + tb2.close()
611 +
612 + logging.debug("compare_CASA_tables(referencetab = {}, testtab = {}): {}".format(referencetab,testtab, rval))
613 + return rval
614 +
615 +def compare_files(file1, file2, shallow=False):
616 + '''
617 + compare_files - Compare two Files.
618 + @param file1 --> a reference file
619 + @param file2 --> a file to verify
620 + @param shallow --> If shallow is true, files with identical os.stat() signatures are taken to be equal. Otherwise, the contents of the files are compared.
621 +
622 + @return: True if file1 & file2 seem equal, False otherwise
623 + '''
624 + logging.info("Comparing {} to {}".format(file1, file2))
625 + logging.debug("Executing: compare_files(file1 = {}, file2 = {}, shallow = {})".format(file1, file2, shallow))
626 + if (sys.version_info > (3,0)):
627 + filecmp.clear_cache()
628 + return filecmp.cmp(file1, file2, shallow=shallow)
629 +
630 +def compare_caltables(table1, table2, cols=[], rtol=8e-7, atol=1e-8):
631 + '''
632 + compare_caltables - Compare two caltables.
633 + @param table1 --> a reference table
634 + @param table2 --> a table to verify
635 + @param cols --> the name of cols to compare (list). Leave Blank For All
636 + @param rtol --> The relative tolerance parameter
637 + @param atol --> The absolute tolerance parameter
638 +
639 + @return: True if table1 == table2 else False
640 + '''
641 + logging.info("Comparing {} to {}".format(table1, table2))
642 + logging.debug("Executing: compare_caltables(table1 = {}, table2 = {}, cols={}, rtol={}, atol={})".format(table1, table2, cols, rtol, atol))
643 + tableVal1 = {}
644 + tableVal2 = {}
645 +
646 + tb.open(table1)
647 + colname1 = tb.colnames()
648 +
649 + for col in colname1:
650 + try:
651 + tableVal1[col] = tb.getcol(col)
652 + except RuntimeError:
653 + pass
654 + tb.close()
655 +
656 + tb2.open(table2)
657 + colname2 = tb2.colnames()
658 +
659 + for col in colname2:
660 + try:
661 + tableVal2[col] = tb2.getcol(col)
662 + except RuntimeError:
663 + pass
664 + tb2.close()
665 +
666 + truthDict = {}
667 +
668 +
669 + for col in tableVal1.keys():
670 + logging.debug("Column: {}, dtype: {}".format(col, tableVal1[col].dtype))
671 + try:
672 + if numpy.issubdtype(tableVal1[col].dtype, numpy.number):
673 + truthDict[col] = numpy.isclose(tableVal1[col], tableVal2[col], rtol=rtol, atol=atol)
674 + else:
675 + # Compare Non Numeric Types
676 + truthDict[col] = numpy.array_equal(tableVal1[col],tableVal2[col])
677 + except:
678 + print(col, 'ERROR in finding truth value')
679 + casalog.post(message=col+': ERROR in determining the truth value')
680 +
681 + if len(cols) == 0:
682 + truths = [[x, numpy.all(truthDict[x] == True)] for x in truthDict.keys()]
683 + else:
684 + truths = [[x, numpy.all(truthDict[x] == True)] for x in cols]
685 +
686 + #Check that All Options are True
687 + for key in truthDict.keys():
688 + if isinstance(truthDict[key], bool):
689 + if not truthDict[key]:
690 + logging.info("{0} in caltables do not match".format(key))
691 + return False
692 + elif isinstance(truthDict[key], numpy.ndarray):
693 + if not numpy.all(truthDict[key]):
694 + return False
695 + else:
696 + logging.info('ERROR in finding truth value for Column: {}'.format(key))
697 + return False
698 +
699 + return True
700 +
701 +def compare_dictionaries( dictionary1, dictionary2, skipkeys = [], rtol=8e-7, atol=1e-8):
702 + '''
703 + compare_dictionaries - compare two dictionaries
704 + Dictionaries will fail when 1st instance of a failure
705 + @param dictionary1 --> the dictionary which is assumed to be correct
706 + @param dictionary2 --> the dictionary which is to be compared
707 + @param skipkeys --> list of keys which are to be ignored
708 + @param rtol --> The relative tolerance parameter
709 + @param atol --> The absolute tolerance parameter
710 +
711 + @return: True if dictionary1 == dictionary2 else False
712 + '''
713 + if not isinstance(skipkeys, list):
714 + logging.error("skipkeys not in correct format")
715 + raise TypeError("skipkeys must be a list")
716 +
717 + key_list_1 = sorted(list(dictionary1.keys()))
718 + key_list_2 = sorted(list(dictionary2.keys()))
719 +
720 + #Checks if Keys are the same
721 + if key_list_1 != key_list_2:
722 + logging.debug("Keys Do Not Match")
723 + return False
724 +
725 + for key in key_list_1:
726 + if key in skipkeys:
727 + continue
728 + # Compare Numpy Arrays
729 + if isinstance(dictionary1[key], numpy.ndarray) and isinstance(dictionary2[key], numpy.ndarray):
730 + """
731 + For finite values, isclose uses the following equation to test whether two floating point values are equivalent.
732 + absolute(a - b) <= (atol + rtol * absolute(b))
733 + """
734 + if numpy.issubdtype(dictionary1[key].dtype, numpy.number) and numpy.issubdtype(dictionary2[key].dtype, numpy.number):
735 + if any( val == False for val in numpy.isclose(dictionary1[key], dictionary2[key], rtol=rtol, atol=atol, equal_nan=False)):
736 + logging.info("{0}:{1} != {0}:{2}".format(key,dictionary1[key],dictionary2[key]))
737 + return False
738 + else:
739 +
740 + if any( val == False for val in numpy.array_equal(dictionary1[key], dictionary2[key])):
741 + logging.info("{0}:{1} != {0}:{2}".format(key,dictionary1[key],dictionary2[key]))
742 + return False
743 +
744 + # Compare Strings
745 + elif isinstance(dictionary1[key], six.string_types) and isinstance(dictionary2[key], six.string_types):
746 +
747 + if (dictionary1[key] == dictionary2[key]):
748 + pass
749 + else:
750 + logging.info("{0}:{1} != {0}:{2}".format(key,dictionary1[key],dictionary2[key]))
751 + return False
752 +
753 + # Compare lists
754 + elif isinstance(dictionary1[key], list) and isinstance(dictionary2[key], list):
755 +
756 + if dictionary1[key] != dictionary2[key]:
757 + logging.info("{0}:{1} != {0}:{2}".format(key,dictionary1[key],dictionary2[key]))
758 + return False
759 +
760 + # Compare Numerics
761 + elif isinstance(dictionary1[key], numbers.Number) and isinstance(dictionary2[key], numbers.Number):
762 + """
763 + rel_tol is the relative tolerance : it is the maximum allowed difference between a and b, relative to the larger absolute value of a or b.
764 + For example, to set a tolerance of 5%, pass rel_tol=0.05. The default tolerance is 1e-09, which assures that the two values are the same within about 9 decimal digits. rel_tol must be greater than zero.
765 +
766 + abs_tol is the minimum absolute tolerance : useful for comparisons near zero. abs_tol must be at least zero.
767 +
768 + If no errors occur, the result will be: abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol).
769 + """
770 +
771 + if not numpy.isclose(dictionary1[key],dictionary2[key],rtol = rtol, atol=atol):
772 + logging.info("{0}:{1} != {0}:{2}".format(key,dictionary1[key],dictionary2[key]))
773 + return False
774 + else:
775 + try:
776 + if dictionary1[key] != dictionary2[key]:
777 + return False
778 + except:
779 + logging.error("Error in Comparing {0}:{1} != {0}:{2}".format(key,dictionary1[key],dictionary2[key]))
780 + return False
781 + return True
782 +
783 +def compare_directories( directory1, directory2):
784 + '''
785 + Compare two directories recursively. Files in each directory are
786 + assumed to be equal if their names and contents are equal.
787 +
788 + @param directory1: First directory path
789 + @param directory2: Second directory path
790 +
791 + @return: True if the directory trees are the same and
792 + there were no errors while accessing the directories or files,
793 + False otherwise.
794 + '''
795 + dirs_cmp = filecmp.dircmp(directory1, directory2)
796 + if len(dirs_cmp.left_only)>0 or len(dirs_cmp.right_only)>0 or \
797 + len(dirs_cmp.funny_files)>0:
798 + return False
799 + (_, mismatch, errors) = filecmp.cmpfiles(
800 + directory1, directory2, dirs_cmp.common_files, shallow=False)
801 + if len(mismatch)>0 or len(errors)>0:
802 + return False
803 + for common_dir in dirs_cmp.common_dirs:
804 + new_directory1 = os.path.join(directory1, common_dir)
805 + new_directory2 = os.path.join(directory2, common_dir)
806 + if not compare_directories(new_directory1, new_directory2):
807 + return False
808 + return True
809 +
810 +def check_pixels(imagename='', loc=None, refval=None, rtol=1e-05, atol=1e-08):
811 + '''
812 + Check pixels in an image to a specified reference value
813 +
814 + @param imagename: input image file
815 + @param loc: The index of the image to compare to the refval
816 + @param refval: The reference value to compare the selected pixel(s) to
817 + @param rtol: The relative tolerance used in the numpy.isclose function
818 + @param atol: The absolute tolerance used in the numpy.isclose function
819 +
820 + @return: True if the shape and value of the refval and selected pixel match.
821 +
822 + '''
823 + if not isinstance(loc,six.string_types):
824 + raise TypeError('Please give target location in string list format "20,30,2:4"')
825 +
826 + if os.path.exists(imagename):
827 + tb.open(imagename)
828 + image = tb.getcol('map')
829 + tb.close()
830 +
831 + if type(refval) != type(None):
832 +
833 + index = []
834 + to_slice = loc.split(',')
835 +
836 + for item in to_slice:
837 + if ':' not in item:
838 + index.append(int(item))
839 + else:
840 + item_split = item.split(':')
841 + index.append(slice(int(item_split[0]),int(item_split[1])))
842 +
843 + selected_slice = image[tuple(index)]
844 +
845 + if numpy.shape(selected_slice) != numpy.shape(refval):
846 + logging.warning('Please check that the shape of the reference and selected slice are the same')
847 + return False
848 +
849 + isequal = numpy.isclose(selected_slice, refval, rtol=rtol, atol=atol)
850 +
851 + logging.info("For pixel value check the obtained value was {}. The expected value was {} with a tolerance of {}. test success = {}.".format(selected_slice, refval, atol, isequal))
852 + return numpy.all(isequal == True)
853 +
854 + else:
855 + logging.warning('Please provide a refernce value to compare against')
856 +
857 + else:
858 + logging.warning('Not a valid Image name')
859 +
860 +def compare_pixel_value(imagename=None, refimage=None, loc=None):
861 + '''
862 + Compare two images at a certain reference pixel
863 +
864 + @param imagename: Name of the image to be compared to a reference image
865 + @param refimage: Image to be compared against
866 + @param loc: The slice or pixel index to compare between the two images
867 +
868 + @return: True if the pixel values match at the provided index or slice. Returns False otherwise
869 + '''
870 + if imagename != None and refimage != None:
871 +
872 + if type(loc) == type(''):
873 +
874 + tb.open(imagename)
875 + image1 = tb.getcol('map')
876 + tb.close()
877 +
878 + tb.open(refimage)
879 + image2 = tb.getcol('map')
880 + tb.close()
881 +
882 + index = []
883 + to_slice = loc.split(',')
884 + # get index from the string array
885 + for item in to_slice:
886 + if ':' not in item:
887 + index.append(int(item))
888 + else:
889 + item_split = item.split(':')
890 + index.append(slice(int(item_split[0]),int(item_split[1])))
891 +
892 + selected_slice1 = image1[tuple(index)]
893 + selected_slice2 = image2[tuple(index)]
894 +
895 + isequal = numpy.isclose(selected_slice1, selected_slice2, rtol=1e-05, atol=1e-08)
896 + return numpy.all(isequal == True)
897 +
898 + else:
899 + logging.warning('Please give target location in string list format ("20,30,2:4")')
900 + else:
901 + logging.warning('Please provide both an image and reference image')
902 +
903 +def compare_pixel_mask(maskname='', refmask=None, refval=None, loc=None):
904 + '''
905 + Compare to masks or mask values to a reference value
906 +
907 + @param maskname: The name of the maskfile to compare to either a reference mask file or value
908 + @param refmask: The reference mask image to be compared to
909 + @param refval: The reference value to compare the selected pixel(s) of the maskfile to
910 + @param loc: The index or slice of the mask image to compare to a refvalue.
911 +
912 + @return: True if the refmask and mask file are identical or if the selected slice of the mask file matches the refval
913 + '''
914 +
915 + if os.path.exists(maskname):
916 + if refmask == None and refval == None:
917 + logging.warning('Please select a mask or region to use for comparison')
918 +
919 + elif refmask != None and refval == None:
920 + # if comparing a refmask compare the values in the table
921 + if os.path.exists(refmask):
922 + tb.open(maskname)
923 + mask1 = tb.getcol('PagedArray')
924 + tb.close()
925 +
926 + tb.open(refmask)
927 + mask2 = tb.getcol('PagedArray')
928 + tb.close()
929 +
930 + return np.all(mask1 == mask2)
931 +
932 + else:
933 + logging.warning('Invalid refmask file name')
934 +
935 + elif refmask == None and refval != None:
936 + # If using a reference value compare the value/shape to the selected slice
937 + if type(loc) == type(''):
938 +
939 + tb.open(maskname)
940 + image = tb.getcol('PagedArray')
941 + tb.close()
942 +
943 + index = []
944 + to_slice = loc.split(',')
945 + # get index from the string array
946 + for item in to_slice:
947 + if ':' not in item:
948 + index.append(int(item))
949 + else:
950 + item_split = item.split(':')
951 + index.append(slice(int(item_split[0]),int(item_split[1])))
952 +
953 + selected_slice = image[tuple(index)]
954 + # return false if the shapes don't match up
955 + if numpy.shape(selected_slice) != numpy.shape(refval):
956 + logging.warning('Please check that the shape of the reference and selected slice are the same')
957 + return False
958 +
959 + isequal = numpy.all(selected_slice == refval)
960 +
961 + return isequal
962 +
963 + else:
964 + logging.warning('Please give target location in string list format ("20,30,2:4")')
965 + else:
966 + logging.warning('Please provide only a referance value or reference mask, not both')
967 + else:
968 + logging.warning('Invalid mask file name')
969 +
970 +
971 +def add_to_dict(self, output=None, dataset="TestData", status=False, **kwargs):
972 + '''
973 + This function adds key value pairs to a provided dictionary. Any additional keys and values can be added as keyword arguments to this function
974 +
975 + @param output: This is the dictionary that the key-value pairs will be appended to
976 + @param filename: This is the name of the test script file
977 + @param dataset: This is the name of the dataset used when executing this test case
978 +
979 + @return: Nothing is returned, the output dict is modified by this function
980 + '''
981 + import inspect
982 + frame = inspect.stack()[1]
983 + module = inspect.getmodule(frame[0])
984 + filename = module.__file__
985 +
986 + testcase = unittest.TestCase.id(self)
987 + test_split = testcase.split('.')
988 + test_case = test_split[-1]
989 + taskname = test_split[1].split('_')[0]
990 +
991 + if (sys.version_info > (3, 3)):
992 + rerun = "python {} {}.{}".format(filename, test_split[1], test_split[2])
993 + else:
994 + filename = "{}.py".format(filename.split('.')[0])
995 + casapath = os.environ.get('CASAPATH').split()[0]
996 + rerun = "{}/bin/casa -c {}/lib/python2.7/runUnitTest.py {}".format(casapath,casapath, filename.split('.')[0])
997 +
998 + current_case = None
999 +
1000 + func_calls = []
1001 + values = {key:kwargs[key] for key in kwargs}
1002 +
1003 + with open(filename, 'r') as file:
1004 + for line in file:
1005 + line = line.strip()
1006 + if line.startswith('def test_'):
1007 + if line.split()[1][:-7].endswith(test_case):
1008 + current_case = test_case
1009 + else:
1010 + current_case = None
1011 + #casa6tasks, miscellaneous_tasks
1012 + for i in casa6tasks.union(miscellaneous_tasks):
1013 + if current_case == test_case:
1014 + if "{}(".format(i) in line:
1015 + params = line.split(',')[1::]
1016 + call = "{}({},{})".format(taskname, dataset, ','.join(params))
1017 + #func_calls.append(call)
1018 + func_calls.append(line)
1019 +
1020 + values['runtime'] = -1.0
1021 + #This is a temp error value
1022 + values['status'] = status
1023 + if test_case not in output.keys():
1024 + output[test_case]= {}
1025 +
1026 + for key in values.keys():
1027 + if test_case in output.keys():
1028 + #print("output[test_case].keys(): {}".format(output[test_case].keys()))
1029 + if key in output[test_case].keys():
1030 + values[key] = output[test_case][key].append(values[key])
1031 + else:
1032 +
1033 + output[test_case][key] = [values[key]]
1034 +
1035 + #output[test_case] = values
1036 + output[test_case]['taskcall'] = func_calls
1037 + output[test_case]['rerun'] = rerun
1038 + output[test_case]['description'] = unittest.TestCase.shortDescription(self)
1039 + output[test_case]['images'] = [ ]
1040 +
1041 + #print("Test Case: {}".format(test_case))
1042 + #print("{} : {}".format(test_case,output[test_case]))
1043 +
1044 +def topickle(input_dict, picklefile):
1045 + '''
1046 + Add a new dictionary into the existing pickle file
1047 +
1048 + @param input_dict: The dictionary object to add to the pickle file
1049 + @param picklefile: The picklefile containing a dictionary to be appended to
1050 +
1051 + @return: Nothing is returned by this function
1052 + '''
1053 + pickle_read = open(picklefile, 'rb')
1054 + pickle_dict = pickle.load(pickle_read)
1055 + # Make sure that the pickle file contains a dictionary
1056 + if type(pickle_dict) != type({}):
1057 + logging.warning('The pickle file is not a dictionary')
1058 + # Add to the dictionary in the pickle file
1059 + for item in list(input_dict.keys()):
1060 + pickle_dict[item] = input_dict[item]
1061 + # Re-write the pickle file with the new dictionary
1062 + with open(picklefile, 'wb') as fout:
1063 + pickle.dump(pickle_dict, fout)
1064 +
1065 +def default_CASA_tasks():
1066 + '''
1067 + default_CASA_tasks - Default Casa Tasks
1068 + Delete all *.last files and restore tasks to default
1069 +
1070 + Returns
1071 + '''
1072 + logging.debug("Executing: default_CASA_tasks")
1073 + # Get a list of all files in directory
1074 + for rootDir, subdirs, filenames in os.walk(os.getcwd()):
1075 + # Find the files that matches the given patterm
1076 + for filename in fnmatch.filter(filenames, '*.last'):
1077 + try:
1078 + os.remove(os.path.join(rootDir, filename))
1079 + except OSError:
1080 + logging.error("Error while deleting file")
1081 + if casa5:
1082 + for task in casa6tasks:
1083 + logging.debug("Defaulting Task: {}".format(task))
1084 + default(task)
1085 + for task in miscellaneous_tasks:
1086 + logging.debug("Defaulting Task: {}".format(task))
1087 + default(task)
1088 + return
1089 +
1090 +def get_directory_size(directory):
1091 + '''
1092 + get_directory_size - Return the size of a directory in bytes
1093 + directory --> the directory which is to be summed
1094 +
1095 + Returns Return the size, in bytes, of directory
1096 + '''
1097 + logging.debug("Executing: get_directory_size(directory = {})".format(directory))
1098 + total_size = 0
1099 + for dirpath, dirnames, filenames in os.walk(directory):
1100 + for filename in filenames:
1101 + fp = os.path.join(dirpath, filename)
1102 + total_size += os.path.getsize(fp)
1103 + logging.debug("Directory: {}, Size: {} Bytes ( {}MB, {}GB) ".format(directory, total_size, (total_size/(1024.0**2)), float(total_size/(1024.0**3))))
1104 + return total_size
1105 +
1106 +def get_table_column(table, colname):
1107 + '''Return the requested variable column
1108 + table --> name of table or MS
1109 + colname --> column name
1110 + Return the column as a dictionary
1111 + '''
1112 +
1113 + col = {}
1114 + tb.open(table)
1115 + if tb.isvarcol(colname):
1116 + col = tb.getvarcol(colname)
1117 + else:
1118 + logging.error("Error Returning Column {}".format(colname))
1119 + return None
1120 +
1121 + tb.close()
1122 + return col
1123 +
1124 +
1125 +def get_caltable_column(caltable, colname='CPARAM'):
1126 + ''' Open a caltable and get the provided column
1127 + caltable --> name of cal table
1128 + colname --> column name
1129 + Return the column as a dictionary
1130 + '''
1131 + tb.open(caltable)
1132 + outtable = tb.getcol(colname)
1133 + tb.close()
1134 + return outtable
1135 +
1136 +def get_column_shape(tab,col,start_row=0,nrow=1,row_inc=1):
1137 + '''
1138 + Get the shape of the given column.
1139 + Keyword arguments:
1140 + tab -- input table or MS
1141 + col -- column to get the shape
1142 + start_row -- start row (default 0)
1143 + nrow -- number of rows to read (default 1)
1144 + row_inc -- increment of rows to read (default 1)
1145 +
1146 + Return a list of strings with the shape of each row in the column.
1147 +
1148 + '''
1149 +
1150 + col_shape = []
1151 + try:
1152 + try:
1153 + tb.open(tab)
1154 + col_shape = tb.getcolshapestring(col,start_row,nrow,row_inc)
1155 + except:
1156 + print('Cannot get shape of col {} from table {} '.format(col,tab))
1157 +
1158 + finally:
1159 + tb.close()
1160 +
1161 + return col_shape
1162 +
1163 +def check_plotfile(plotfileName, min_size, max_size=None):
1164 + '''
1165 + Check if plotfile generated is cprrect size
1166 + plotfileName --> Name of plotted Image
1167 + min_size -- > Min Size of image
1168 + max_size --> Max Size of image
1169 +
1170 + Return : True if image size > min_size ( and < max_size if max_size is provided )
1171 + '''
1172 + val = False
1173 + if os.path.isfile(plotfileName):
1174 + plotSize = os.path.getsize(plotfileName) # Return the size, in bytes, of path.
1175 + logging.info( '{} file size is: {}'.format( plotfileName, plotSize))
1176 + if plotSize > min_size:
1177 + val = True
1178 + if max_size is not None:
1179 + if not plotSize < max_size:
1180 + val = False
1181 +
1182 + else:
1183 + logging.critical("Plot was not created")
1184 +
1185 + return val
1186 +
1187 +def generate_weblog(task,dictionary):
1188 + """Generate Test Summary Weblog
1189 +
1190 + Example:
1191 + generate_weblog("taskname", dictionary)
1192 + """
1193 + global html
1194 + html = open("test_{}_weblog.html".format(task.lower()), 'w')
1195 + Weblog(task, dictionary).generate_weblog()
1196 + html.close()
1197 +
1198 +############################################################################################
1199 +################################## imagerhelpers ###############################
1200 +############################################################################################
1201 +
1202 +def check_model(msname=""):
1203 + logging.debug("Executing: check_model(msname={})".format(msname))
1204 + hasmodcol = False
1205 + modsum=0.0
1206 + hasvirmod = False
1207 +
1208 + tb.open( msname )
1209 + hasmodcol = ( (tb.colnames()).count('MODEL_DATA')>0 )
1210 +
1211 + if hasmodcol:
1212 + model_data = tb.getcol('MODEL_DATA')
1213 + modsum = model_data.sum()
1214 + tb.close()
1215 +
1216 + tb.open( msname+'/SOURCE' )
1217 + keys = tb.getkeywords()
1218 + if len(keys)>0:
1219 + hasvirmod=True
1220 + tb.close()
1221 +
1222 + tb.open( msname )
1223 + keys = tb.getkeywords()
1224 + for key in keys:
1225 + if key.count("model_")>0:
1226 + hasvirmod=True
1227 + tb.close()
1228 +
1229 + logging.info("MS Name: {}, modelcol= {}, modsum = {}, virmod = {}".format( msname, hasmodcol, modsum, hasvirmod ))
1230 +
1231 + return hasmodcol, modsum, hasvirmod
1232 +
1233 +def get_max(imname):
1234 + """Get Image max"""
1235 + logging.debug("Executing: get_max(imname={})".format(imname))
1236 + ia.open(imname)
1237 + stat = ia.statistics()
1238 + ia.close()
1239 + logging.debug("stat['max'] = {}".format(stat['max']))
1240 + logging.debug("stat['maxpos'] = {}".format(stat['maxpos']))
1241 + return stat['max'],stat['maxpos']
1242 +
1243 +def get_pix(imname,pos):
1244 + """Get Image val"""
1245 + ia.open(imname)
1246 + apos = ia.pixelvalue(pos)
1247 + ia.close()
1248 + if apos == {}:
1249 + return None
1250 + else:
1251 + return apos['value']['value']
1252 +
1253 +def get_pixmask(imname,pos):
1254 + """Get Image Mask val"""
1255 + ia.open(imname)
1256 + apos = ia.pixelvalue(pos)
1257 + ia.close()
1258 +
1259 + if apos == {}:
1260 + return None
1261 + else:
1262 + return apos['mask']
1263 +
1264 +def check_beam_compare(image1, image2, op=operator.le):
1265 + """Compare all plane of cube beam image1 operator op than image1"""
1266 + ia.open(image1)
1267 + nchan = ia.shape()[3]
1268 + beam1 = numpy.zeros(nchan)
1269 + for k in range(nchan):
1270 + beam1[k]= ia.beamarea(k,0)['arcsec2']
1271 + ia.close()
1272 +
1273 + ia.open(image2)
1274 + if(nchan != ia.shape()[3]):
1275 + return False
1276 + beam2 = numpy.zeros(nchan)
1277 + for k in range(nchan):
1278 + beam2[k] = ia.beamarea(k,0)['arcsec2']
1279 + ia.close()
1280 +
1281 + return numpy.alltrue(op(beam1, beam2))
1282 +
1283 +def exists(imname):
1284 + """ Image exists """
1285 + return os.path.exists(imname)
1286 +
1287 +def get_peak_res(summ):
1288 + if summ.has_key('summaryminor'):
1289 + reslist = summ['summaryminor'][1,:]
1290 + peakres = reslist[ len(reslist)-1 ]
1291 + else:
1292 + peakres = None
1293 + return peakres
1294 +
1295 +
1296 +def check_peak_res(summ,correctres, epsilon=0.05):
1297 +
1298 + peakres = get_peak_res(summ)
1299 + out = True
1300 + if correctres == None and peakres != None:
1301 + out = False
1302 + return out,peakres
1303 + if correctres != None and peakres == None:
1304 + out = False
1305 + return out,peakres
1306 +
1307 + if out==True and peakres != None:
1308 + if abs(correctres - peakres)/abs(correctres) > epsilon:
1309 + out=False
1310 + return out,peakres
1311 + return out,peakres
1312 +
1313 +def get_mod_flux(summ):
1314 + if summ.has_key('summaryminor'):
1315 + modlist = summ['summaryminor'][2,:]
1316 + modflux = modlist[ len(modlist)-1 ]
1317 + else:
1318 + modflux = None
1319 +
1320 + return modflux
1321 +
1322 +def check_mod_flux(summ,correctmod, epsilon=0.05):
1323 + modflux = get_mod_flux(summ)
1324 + out = True
1325 + if correctmod == None and modflux != None:
1326 + out = False
1327 + return out,peakres
1328 + if correctmod != None and modflux == None:
1329 + out = False
1330 + return out,peakres
1331 + if out==True and modflux != None:
1332 + if abs(correctmod - modflux)/abs(correctmod) > epsilon:
1333 + out=False
1334 + return out,peakres
1335 + return out,modflux
1336 +
1337 +def get_iter_done(summ):
1338 + if summ.has_key('iterdone'):
1339 + iters = summ['iterdone']
1340 + else:
1341 + iters = None
1342 + return iters
1343 +
1344 +def verdict(boolval):
1345 + return "Pass" if boolval else "Fail"
1346 +
1347 +def check_ret( summ,correctres,correctmod,epsilon = 0.05):
1348 + pstr = ''
1349 + if casa5:
1350 + testname = inspect.stack()[1][3] # Make Sure this is correct
1351 + else:
1352 + testname = "TODO"
1353 + retres, peakres = check_peak_res(summ, correctres, epsilon)
1354 + retmod, modflux = check_mod_flux(summ, correctmod, epsilon)
1355 +
1356 + pstr_peak = "[ {} ] PeakRes is {} ( {} : should be {} + )\n".format(testname, str(peakres), verdict(retres) , str(correctres))
1357 + pstr_mod = "[ {} ] Modflux is {} ( {} : should be {} + )\n".format(testname, str(modflux), verdict(retmod) , str(correctmod))
1358 + pstr = pstr_peak + pstr_mod
1359 + logging.info(pstr)
1360 + if retres==False or retmod==False:
1361 + return False, pstr
1362 + else:
1363 + return True, pstr
1364 +
1365 +def check_val(val, correctval, valname='Value', exact=False, epsilon=0.05):
1366 + pstr = ''
1367 + if casa5:
1368 + testname = inspect.stack()[2][3] # Make Sure this is correct
1369 + else:
1370 + testname = "TODO"
1371 +
1372 + out = True
1373 +
1374 + if numpy.isnan(val) or numpy.isinf(val):
1375 + out=False
1376 + if correctval == None and val != None:
1377 + out = False
1378 + if correctval != None and val == None:
1379 + out = False
1380 + if out==True and val != None:
1381 + if exact==True:
1382 + if correctval != val:
1383 + out=False
1384 + else:
1385 + if abs(correctval - val)/abs(correctval) > epsilon:
1386 + out=False
1387 +
1388 + pstr = "[ {} ] {} is {} ( {} : should be {} )\n".format(testname, valname, str(val), verdict(out), str(correctval) )
1389 +
1390 + logging.info(pstr)
1391 + return out, pstr
1392 +
1393 +def check_val_less_than(val, bound, valname='Value'):
1394 + pstr = ''
1395 + if casa5:
1396 + testname = inspect.stack()[2][3] # Make Sure this is correct
1397 + else:
1398 + testname = "TODO"
1399 +
1400 + out = True
1401 +
1402 + if numpy.isnan(val) or numpy.isinf(val):
1403 + out=False
1404 + if bound == None and val != None:
1405 + out = False
1406 + if bound != None and val == None:
1407 + out = False
1408 + if out==True and val != None:
1409 + if val > bound:
1410 + out=False
1411 +
1412 + pstr = "[ {} ] {} is {} ( {} : should be less than {} )\n".format(testname, valname, str(val), verdict(out), str(bound))
1413 +
1414 + logging.info(pstr)
1415 + return out, pstr
1416 +
1417 +def check_val_greater_than(val, bound, valname='Value'):
1418 + pstr = ''
1419 + if casa5:
1420 + testname = inspect.stack()[2][3] # Make Sure this is correct
1421 + else:
1422 + testname = "TODO"
1423 +
1424 + out = True
1425 +
1426 + if numpy.isnan(val) or numpy.isinf(val):
1427 + out=False
1428 + if bound == None and val != None:
1429 + out = False
1430 + if bound != None and val == None:
1431 + out = False
1432 + if out==True and val != None:
1433 + if val < bound:
1434 + out=False
1435 +
1436 + pstr = "[ {} ] {} is {} ( {} : should be greater than {} )\n".format(testname, valname, str(val), verdict(out), str(bound))
1437 +
1438 + logging.info(pstr)
1439 + return out, pstr
1440 +
1441 +def check_ims(imlist,truth):
1442 + if casa5:
1443 + testname = inspect.stack()[2][3]
1444 + else:
1445 + testname = "TODO"
1446 +
1447 + imex=[]
1448 + out=True
1449 +
1450 + for imname in imlist:
1451 + ondisk = exists(imname)
1452 + imex.append( ondisk )
1453 + if ondisk != truth:
1454 + out=False
1455 +
1456 + pstr = "[ {} ] Image made : {} = {} ( {} : should all be {} )\n".format(testname, str(imlist), str(imex), verdict(out),str(truth))
1457 + logging.info(pstr)
1458 + return pstr
1459 +
1460 +def check_keywords(imlist):
1461 + """
1462 + Keyword related checks (presence/absence of records and entries in these records,
1463 + in the keywords of the image table).
1464 +
1465 + :param imlist: names of the images produced by a test execution.
1466 +
1467 + :returns: the usual (test_imager_helper) string with success/error messages.
1468 + """
1469 + # Keeping the general approach. This is fragile!
1470 + if casa5:
1471 + testname = inspect.stack()[2][3]
1472 + else:
1473 + testname = "TODO"
1474 +
1475 + # accumulator of error strings
1476 + pstr = ''
1477 + for imname in imlist:
1478 + if os.path.exists(imname):
1479 + issues = check_im_keywords(imname, check_misc=True, check_extended=True)
1480 + if issues:
1481 + pstr += '[{0}] {1}: {2}'.format(testname, imname, issues)
1482 +
1483 + if not pstr:
1484 + pstr += 'All expected keywords in imageinfo, miscinfo, and coords found.\n'
1485 + return pstr
1486 +
1487 +def check_im_keywords(imname, check_misc=True, check_extended=True):
1488 + """
1489 + Checks several lists of expected and forbidden keywords and entries of these
1490 + keywords.
1491 + Forbidden keywords lists introduced with CAS-9231 (prevent duplication of
1492 + TELESCOP and OBJECT).
1493 +
1494 + Note that if imname is the top level of a refconcat image, there's no table to open
1495 + to look for its keywords. In these cases nothing is checked. We would not have the
1496 + 'imageinfo' keywords, only the MiscInfo that goes in imageconcat.json and I'm not
1497 + sure yet how that one is supposed to behave.
1498 + Tests should check the 'getNParts() from imname' to make sure the components of
1499 + the refconcat image exist, have the expected keywords, etc.
1500 +
1501 + :param imname: image name (output image from tclean)
1502 + :param check_misc: whether to check miscinfo in addition to imageinfo'
1503 + :param check_extended: can leave enabled for images other than .tt?, .alpha, etc.
1504 +
1505 + :returns: the usual (test_imager_helper) string with success/error messages.
1506 + Errors marked with '(Fail' as per self.verdict().
1507 + """
1508 +
1509 + try:
1510 + tbt.open(imname)
1511 + keys = tbt.getkeywords()
1512 + except RuntimeError as exc:
1513 + if os.path.isfile(os.path.join(os.path.abspath(imname), 'imageconcat.json')):
1514 + # Looks like a refconcat image, nothing to check
1515 + #return ''
1516 + # make a bit more informative
1517 + pstr = 'Looks like it is a refconcat image. Skipping the imageinfo keywords check.\n'
1518 + return pstr
1519 + else:
1520 + pstr = 'Cannot open image table to check keywords: {0}\n'.format(imname)
1521 + return pstr
1522 + finally:
1523 + tbt.close()
1524 +
1525 + pstr = ''
1526 + if len(keys) <= 0:
1527 + pstr += ('No keywords found ({0})\n'.format(verdict(False)))
1528 + return pstr
1529 +
1530 + # Records that need to be present
1531 + imageinfo = 'imageinfo'
1532 + miscinfo = 'miscinfo'
1533 + coords = 'coords'
1534 + mandatory_recs = [imageinfo, coords]
1535 + if check_misc:
1536 + mandatory_recs.append(miscinfo)
1537 + for rec in mandatory_recs:
1538 + if rec not in keys:
1539 + pstr += ('{0} record not found ({1})\n'.format(rec, verdict(False)))
1540 +
1541 + if len(pstr) > 0:
1542 + return pstr
1543 +
1544 + mandatory_imageinfo = ['objectname', 'imagetype']
1545 + pstr += check_expected_entries(mandatory_imageinfo, imageinfo, keys)
1546 +
1547 + if check_misc:
1548 + if check_extended:
1549 + mandatory_miscinfo = ['INSTRUME', 'distance']
1550 + pstr += check_expected_entries(mandatory_miscinfo, miscinfo, keys)
1551 + forbidden_miscinfo = ['OBJECT', 'TELESCOP']
1552 + pstr += check_forbidden_entries(forbidden_miscinfo, miscinfo, keys)
1553 +
1554 + mandatory_coords = ['telescope']
1555 + pstr += check_expected_entries(mandatory_coords, coords, keys)
1556 +
1557 + return pstr
1558 +
1559 +def check_expected_entries( entries, record, keys):
1560 + pstr = ''
1561 + for entry in entries:
1562 + if entry not in keys[record]:
1563 + pstr += ('entry {0} not found in record {1} ({2})\n'.format(entry, record, verdict(False)))
1564 + else:
1565 + # TODO: many tests leave 'distance' empty. Assume that's acceptable...
1566 + if entry != 'distance' and not keys[record][entry]:
1567 + pstr += ('entry {0} is found in record {1} but it is empty ({2})\n'.format(entry, record, verdict(False)))
1568 + return pstr
1569 +
1570 +def check_forbidden_entries( entries, record, keys):
1571 + pstr = ''
1572 + for entry in entries:
1573 + if entry in keys[record]:
1574 + pstr += ('entry {0} should not be in record {1} ({2})\n'.format(entry, record, verdict(False)))
1575 + return pstr
1576 +
1577 +def check_pix_val(imname,theval=0, thepos=[0,0,0,0], exact=False, epsilon=0.05):
1578 + if casa5:
1579 + testname = inspect.stack()[2][3]
1580 + else:
1581 + testname = "TODO"
1582 +
1583 + readval = get_pix(imname,thepos)
1584 +
1585 + res=True
1586 +
1587 + if readval==None:
1588 + res=False
1589 + elif numpy.isnan(readval) or numpy.isinf(readval):
1590 + res=False
1591 + else:
1592 + if abs(theval) > epsilon:
1593 + if exact==False:
1594 + if abs(readval - theval)/abs(theval) > epsilon:
1595 + res = False
1596 + else:
1597 + res = True
1598 + else:
1599 + if abs(readval - theval) > 0.0:
1600 + res = False
1601 + else:
1602 + res = True
1603 + else: ## this is to guard against exact zero... sort of.
1604 + if abs(readval - theval) > epsilon:
1605 + res = False
1606 + else:
1607 + res = True
1608 +
1609 + pstr = "[ {} ] {} : Value is {} at {} ( {} : should be {} )\n".format(testname, imname, str(readval), str(thepos), verdict(res), str(theval))
1610 + logging.info(pstr)
1611 + return pstr
1612 +
1613 +def check_pixmask(imname,theval=True, thepos=[0,0,0,0]):
1614 + if casa5:
1615 + testname = inspect.stack()[2][3]
1616 + else:
1617 + testname = "TODO"
1618 + readval = get_pixmask(imname,thepos)
1619 +
1620 + res=True
1621 +
1622 + if readval==None:
1623 + res=False
1624 + elif numpy.isnan(readval) or numpy.isinf(readval) or type(readval)!=bool:
1625 + res=False
1626 + else:
1627 + if readval == theval:
1628 + res = True
1629 + else:
1630 + res = False
1631 + pstr = "[ {} ] {} : Mask is {} at {} ( {} : should be {} )\n".format(testname, imname, str(readval), str(thepos), verdict(res), str(theval))
1632 + logging.info(pstr)
1633 + return pstr
1634 +
1635 +def check_ref_freq(imname,theval=0, epsilon=0.05):
1636 + testname = inspect.stack()[2][3]
1637 +
1638 + retres=True
1639 +
1640 + ia.open(imname)
1641 + csys = ia.coordsys()
1642 + ia.close()
1643 +
1644 + reffreq = csys.referencevalue()['numeric'][3]
1645 + if abs(reffreq - theval)/theval > epsilon :
1646 + retres=False
1647 + else:
1648 + retres=True
1649 +
1650 +
1651 + pstr = "[ {} ] Ref-Freq is {} ( {} : should be {} )\n".format(testname , str(reffreq) , verdict(retres), str(theval))
1652 + logging.info(pstr)
1653 + return pstr
1654 +
1655 +###################################
1656 +def check_imexist(imgexist):
1657 + pstr = ''
1658 + if imgexist != None:
1659 + if type(imgexist)==list:
1660 + pstr += check_ims(imgexist, True)
1661 + print("pstr after checkims = {}".format(pstr))
1662 + pstr += check_keywords(imgexist)
1663 + print("pstr after check_keywords = {}".format(pstr))
1664 + return pstr
1665 +
1666 +def check_imexistnot(imgexistnot):
1667 + pstr = ''
1668 + if imgexistnot != None:
1669 + if type(imgexistnot)==list:
1670 + pstr += check_ims(imgexistnot, False)
1671 + return pstr
1672 +
1673 +def check_imval(imgval, epsilon=0.05):
1674 + pstr = ''
1675 + if imgval != None:
1676 + if type(imgval)==list:
1677 + for ii in imgval:
1678 + if type(ii)==tuple and len(ii)==3:
1679 + pstr += check_pix_val(ii[0],ii[1],ii[2],epsilon=epsilon)
1680 + return pstr
1681 +
1682 +def check_imvalexact(imgvalexact, epsilon=0.05):
1683 + pstr = ''
1684 + if imgvalexact != None:
1685 + if type(imgvalexact)==list:
1686 + for ii in imgvalexact:
1687 + if type(ii)==tuple and len(ii)==3:
1688 + pstr += check_pix_val(ii[0],ii[1],ii[2], exact=True,epsilon=epsilon)
1689 + return pstr
1690 +
1691 +def check_immask(imgmask):
1692 + pstr = ''
1693 + if imgmask != None:
1694 + if type(imgmask)==list:
1695 + for ii in imgmask:
1696 + if type(ii)==tuple and len(ii)==3:
1697 + pstr += check_pixmask(ii[0],ii[1],ii[2])
1698 + return pstr
1699 +
1700 +def check_tabcache(tabcache):
1701 + pstr = ''
1702 + if tabcache==True:
1703 + opentabs = tb.showcache()
1704 + if len(opentabs)>0 :
1705 + pstr += "["+inspect.stack()[1][3]+"] " + verdict(False) + ": Found open tables after run \n"
1706 + return pstr
1707 +
1708 +def check_stopcode(stopcode):
1709 + pstr = ''
1710 + if stopcode != None:
1711 + if type(stopcode)==int:
1712 + stopstr = "["+inspect.stack()[1][3]+"] Stopcode is " + str(ret['stopcode']) + " (" + verdict(ret['stopcode']==stopcode) + " : should be " + str(stopcode) + ")\n"
1713 + print(stopstr)
1714 + pstr += stopstr
1715 + return pstr
1716 +
1717 +def check_reffreq(reffreq):
1718 + pstr = ''
1719 + if reffreq != None:
1720 + if type(reffreq)==list:
1721 + for ii in reffreq:
1722 + if type(ii)==tuple and len(ii)==2:
1723 + pstr += check_ref_freq(ii[0],ii[1])
1724 + return pstr
1725 +
1726 +
1727 +def checkall( ret=None, peakres=None, modflux=None, iterdone=None, nmajordone=None, imgexist=None, imgexistnot=None, imgval=None, imgvalexact=None, imgmask=None, tabcache=True, stopcode=None, reffreq=None, epsilon=0.05 ):
1728 + """
1729 + ret=None,
1730 + peakres=None, # a float
1731 + modflux=None, # a float
1732 + iterdone=None, # an int
1733 + nmajordone=None, # an int
1734 + imgexist=None, # list of image names
1735 + imgexistnot=None, # list of image names
1736 + imgval=None, # list of tuples of (imagename,val,pos)
1737 + imgvalexact=None, # list of tuples of (imagename,val,pos)
1738 + imgmask=None, #list of tuples to check mask value
1739 + tabcache=True,
1740 + stopcode=None,
1741 + reffreq=None # list of tuples of (imagename, reffreq)
1742 + """
1743 +
1744 + pstr = ""
1745 +
1746 + if ret != None and type(ret)==dict:
1747 + try:
1748 + if peakres != None:
1749 + pstr += check_val( val=get_peak_res(ret), correctval=peakres, valname="peak res" )
1750 +
1751 + if modflux != None:
1752 + pstr += check_val( val=get_mod_flux(ret), correctval=modflux, valname="mod flux" )
1753 +
1754 + if iterdone != None:
1755 + pstr += check_val( val=ret['iterdone'], correctval=iterdone, valname="iterdone", exact=True )
1756 +
1757 + if nmajordone != None:
1758 + pstr += check_val( val=ret['nmajordone'], correctval=nmajordone, valname="nmajordone", exact=True )
1759 +
1760 + except Exception as e:
1761 + logging.info(ret)
1762 + raise
1763 + logging.info("Epsilon: {}".format(epsilon))
1764 + pstr += check_imexist(imgexist)
1765 + pstr += check_imexistnot(imgexistnot)
1766 + pstr += check_imval(imgval,epsilon=epsilon)
1767 + pstr += check_imvalexact(imgvalexact,epsilon=epsilon)
1768 + pstr += check_immask(imgmask)
1769 + pstr += check_tabcache(tabcache)
1770 + pstr += check_stopcode(stopcode)
1771 + pstr += check_reffreq(reffreq)
1772 +
1773 + return pstr
1774 +
1775 +def check_final(pstr=""):
1776 + if not isinstance(pstr, six.string_types):
1777 + return False
1778 + casalog.post(pstr,'INFO')
1779 + if( pstr.count("Fail") > 0 ):
1780 + return False
1781 + return True
1782 +############################################################################################
1783 +################################## Decorators ##################################
1784 +############################################################################################
1785 +
1786 +#import casaTestHelper
1787 +#@casaTestHelper.skipIfMissingModule
1788 +def skipIfMissingModule(required_module,strict=False):
1789 + '''
1790 + Decorator: skip test if specified module is not avaliable
1791 +
1792 + Example:
1793 + @casaTestHelper.skipIfMissingModule('astropy')
1794 + def test_test(self):
1795 + '''
1796 + import os
1797 + try:
1798 + __import__(required_module)
1799 + flag = True
1800 + except ImportError:
1801 + flag = False
1802 + def deco(function):
1803 + if not CASA6:
1804 + return deco
1805 +
1806 + def wrapper(self, *args, **kwargs):
1807 + if not flag:
1808 + # If there is a strict flag run the tests as normal
1809 + print(sys.argv)
1810 + if strict:
1811 + function(self)
1812 + pass
1813 + else:
1814 + # Module ImportError and no strict flag
1815 + self.skipTest("ModuleNotFoundError: No module named '{}'".format(required_module))
1816 + else:
1817 + function(self)
1818 + return wrapper
1819 + return deco
1820 +
1821 +#import casaTestHelper
1822 +#@casaTestHelper.time_execution
1823 +
1824 +
1825 +def time_execution(out_dict):
1826 + # TODO Ver if this is the better option
1827 + def time_decorator(function):
1828 + '''
1829 + Decorator: time execution of test
1830 +
1831 + Example:
1832 + @casaTestHelper.time_execution
1833 + def test_test(self):
1834 + '''
1835 + @wraps(function)
1836 + def function_timer(*args, **kwargs):
1837 + failed = False
1838 + result = None
1839 + t0 = time.time()
1840 + print(out_dict)
1841 + try:
1842 + result = function(*args, **kwargs)
1843 + except:
1844 + failed=True
1845 + t1 = time.time()
1846 + out_dict[function.__name__]['runtime'] = t1-t0
1847 + casalog.post("Total time running {}: {} seconds".format(function.__name__, str(t1-t0)))
1848 + #out_dict[function.__name__]['status'] = False
1849 + raise
1850 +
1851 + t1 = time.time()
1852 + #print ("Total time running %s: %s seconds" % (function.__name__, str(t1-t0)))
1853 + casalog.post("Total time running {}: {} seconds".format(function.__name__, str(t1-t0)))
1854 + #print('======================================================')
1855 + #print(function.__name__)
1856 + out_dict[function.__name__]['runtime'] = t1-t0
1857 + out_dict[function.__name__]['status'] = True
1858 +
1859 + return result
1860 +
1861 + return function_timer
1862 + return time_decorator
1863 +
1864 +def cpu_usage(out_dict):
1865 + def cpu_decorator(function):
1866 + @wraps(function)
1867 + def function_usage(*args, **kwargs):
1868 + #Temp Fix : CASA 5 Doesnt Have psutil by default
1869 +
1870 + try:
1871 + import psutil
1872 + use_psutil = True
1873 + except ImportError:
1874 + use_psutil = False
1875 + if use_psutil:
1876 + process = psutil.Process(os.getpid())
1877 + snapshot1 = process.memory_info()
1878 + open_files1 = process.open_files()
1879 + num_file_descriptors1 = process.num_fds()
1880 +
1881 + #print ("Function: {}, {} MBs".format(function.__name__, megs1))
1882 + #print ("Function: {}, Open Files: {}".format(function.__name__, open_files1))
1883 + #print ("Function: {}, num_file_descriptors: {}".format(function.__name__, num_file_descriptors1))
1884 +
1885 + result = function(*args, **kwargs)
1886 +
1887 + process = psutil.Process(os.getpid())
1888 + snapshot2 = process.memory_info()
1889 + open_files2 = process.open_files()
1890 + num_file_descriptors2 = process.num_fds()
1891 +
1892 + #print ("Function: {}, {} MBs".format(function.__name__, megs2))
1893 + #print ("Function: {}, Open Files: {}".format(function.__name__, open_files2))
1894 + #print ("Function: {}, num_file_descriptors: {}".format(function.__name__, num_file_descriptors2))
1895 + #print('{:.2f} MB\n'.format(process.memory_info().rss / 1024 / 1024))
1896 +
1897 + #print ("Total Mem Info { }: {:.2f} MB".format(function.__name__,(process.memory_info().rss) / 1024 / 1024 ))
1898 + out_dict[function.__name__]['cpu_usage'] = { "number of file descriptors opened" : num_file_descriptors2 - num_file_descriptors1,
1899 + "Open files" : open_files2,
1900 + "Pre Memory Snapshot (bytes)" : snapshot1,
1901 + "Post Memory Snapshot (bytes)" : snapshot2
1902 + }
1903 + else:
1904 + #TODO: Add methods to get mem snapshots when psutils is not available
1905 + result = function(*args, **kwargs)
1906 + out_dict[function.__name__]['cpu_usage'] = { "number of file descriptors opened" : "Unknown",
1907 + "Open files" : "Unknown",
1908 + "Pre Memory Snapshot (bytes)" : "Unknown",
1909 + "Post Memory Snapshot (bytes)" : "Unknown"
1910 + }
1911 + return result
1912 + return function_usage
1913 + return cpu_decorator
1914 +
1915 +def peakmem(out_dict):
1916 + #TODO: https://pytracemalloc.readthedocs.io/examples.html
1917 + ### NOTE: Only for python3.4+
1918 +
1919 + def mem_decorator(function):
1920 + @wraps(function)
1921 + def function_mem(*args, **kwargs):
1922 + import sys
1923 + if (sys.version_info > (3, 3)):
1924 + import tracemalloc
1925 + tracemalloc.clear_traces()
1926 + tracemalloc.start()
1927 + snapshot1 = tracemalloc.take_snapshot() # Snapshot of traces of memory blocks allocated by Python.
1928 +
1929 + result = function(*args, **kwargs)
1930 +
1931 + snapshot2 = tracemalloc.take_snapshot()
1932 + peakmem = ("{} MiB".format(tracemalloc.get_traced_memory()[1] / 1024 /1024)) #Get the current size and peak size of memory blocks traced by the tracemalloc module as a tuple: (current: int, peak: int)
1933 + tracemalloc.stop()
1934 + top_stats = snapshot2.compare_to(snapshot1, 'lineno') # Compute the differences with an old snapshot.
1935 + out_dict[function.__name__]['peakmem'] = peakmem
1936 + out_dict[function.__name__]['memleaks'] = top_stats[:10] #
1937 + else:
1938 + result = function(*args, **kwargs)
1939 + out_dict[function.__name__]['peakmem'] = "Unknown"
1940 + out_dict[function.__name__]['memleaks'] = "Unknown" #
1941 + return result
1942 + return function_mem
1943 + return mem_decorator
1944 +
1945 +def mem_use_deco(out_dict):
1946 + def mem_decorator(function):
1947 + @wraps(function)
1948 + def function_mem(*args, **kwargs):
1949 + out = subprocess.Popen(['ps','v','-p', str(os.getpid())], stdout=subprocess.PIPE).communicate()[0].split(b'\n')
1950 + vsz_index = out[0].split().index(b'RSS')
1951 + out_start = float(out[1].split()[vsz_index]) / 1024
1952 +
1953 + result = function(*args, **kwargs)
1954 +
1955 + out = subprocess.Popen(['ps','v','-p', str(os.getpid())], stdout=subprocess.PIPE).communicate()[0].split(b'\n')
1956 + vsz_index = out[0].split().index(b'RSS')
1957 + out_end = float(out[1].split()[vsz_index]) / 1024
1958 +
1959 + out_dict[function.__name__]['Mem Use'] = "{} MiB".format(out_end-out_start)
1960 +
1961 + return result
1962 + return function_mem
1963 + return mem_decorator
1964 +
1965 +def stats_dict(out_dict):
1966 + def stats_decorator(function):
1967 + @time_execution(out_dict)
1968 + #@cpu_usage(out_dict)
1969 + #@peakmem(out_dict)
1970 + @mem_use_deco(out_dict)
1971 + @wraps(function)
1972 + def all_wrapped(*args, **kwargs):
1973 + return function(*args, **kwargs)
1974 + return all_wrapped
1975 + return stats_decorator
1976 +
1977 +
1978 +

Everything looks good. We'll let you know here if there's anything you should know about.

Add shortcut