Commits
Ville Suoranta authored cd9dffdff7c Merge
1 - | from __future__ import print_function |
1 + | ######################################################################################################################## |
2 + | ############################################ Imports ############################################# |
3 + | ######################################################################################################################## |
4 + | |
2 5 | import argparse |
3 6 | import os |
4 7 | import shutil |
5 - | import logging |
6 8 | import sys |
7 9 | import traceback |
8 10 | import subprocess |
9 11 | import unittest |
10 12 | import json |
11 13 | import datetime |
12 14 | import platform |
13 - | from testrunner.shell_runner import ShellRunner |
14 - | from testrunner import xvfb_helper |
15 - | from testrunner.xunit import Xunit |
15 + | |
16 + | |
16 17 | default_timeout = 1800 |
17 18 | sys.path.insert(0,'') |
18 19 | |
20 + | ######################################################################################################################## |
21 + | ###################################### Imports / Constants ####################################### |
22 + | ######################################################################################################################## |
23 + | |
24 + | # mem mode variables |
25 + | HAVE_MEMTEST=True |
26 + | MEM = 0 |
27 + | try: |
28 + | import memTest |
29 + | except ImportError: |
30 + | HAVE_MEMTEST = False |
31 + | |
32 + | # cov mode variables |
33 + | HAVE_COVTEST=True |
34 + | COV = 0 |
35 + | try: |
36 + | import coverage |
37 + | except ImportError: |
38 + | HAVE_COVTEST = False |
39 + | |
40 + | # pybot mode variables |
41 + | HAVE_ROBOT = True |
42 + | USE_PYBOT = 0 |
43 + | try: |
44 + | import robot |
45 + | except ImportError: |
46 + | HAVE_ROBOT = False |
47 + | |
48 + | #### PYTEST IMPORT |
49 + | HAVE_PYTEST = True |
50 + | try: |
51 + | import pytest |
52 + | except ImportError: |
53 + | HAVE_PYTEST = False |
54 + | |
55 + | #### NOSE IMPORT |
56 + | HAVE_NOSE = True |
57 + | try: |
58 + | import nose |
59 + | except ImportError: |
60 + | HAVE_NOSE = False |
61 + | |
62 + | IS_CASA6 = False |
63 + | CASA6 = False |
64 + | HAVE_CASA6 = False |
65 + | verbose = False |
66 + | DRY_RUN = False |
67 + | RUN_ALL = False |
68 + | |
69 + | # JIRA BRANCH TO CHECKOUT |
70 + | JIRA_BRANCH = None |
71 + | |
72 + | try: |
73 + | from __main__ import default |
74 + | from tasks import * |
75 + | from taskinit import * |
76 + | except ImportError: |
77 + | CASA6 = True |
78 + | IS_CASA6 = True |
79 + | HAVE_CASA6 = True |
80 + | |
81 + | # Use Nose attribute Functionality |
82 + | RUN_SUBTEST = False |
83 + | |
84 + | # Dry run of Tests |
85 + | DRY_RUN = False |
86 + | |
19 87 | ######################################################################################################################## |
20 88 | ########################################### Functions ############################################ |
21 89 | ######################################################################################################################## |
22 90 | |
23 91 | class casa_test: |
24 92 | def __init__(self, |
25 93 | name, |
26 94 | path, |
27 95 | test_group=None, |
28 96 | test_type=None, |
212 280 | pytest.main(["--collect-only", test]) |
213 281 | |
214 282 | def clean_working_directory(workdir): |
215 283 | |
216 284 | print("Cleaning: {}".format(workdir)) |
217 285 | if os.path.exists(workdir): |
218 286 | try: |
219 287 | os.rmdir(workdir) |
220 288 | except: |
221 289 | shutil.rmtree(workdir) |
222 - | |
223 290 | |
224 291 | def getname(testfile): |
225 292 | '''Get the test name from the command-line |
226 293 | Ex: from test_clean[test1], returns test_clean''' |
227 294 | n0 = testfile.rfind('[') |
228 295 | n1 = testfile.rfind(']') |
229 296 | if n0 != -1: |
230 297 | return testfile[:n0] |
231 298 | |
232 299 | def gettests(testfile): |
303 370 | def unpack_tarball(pkg, outputdir): |
304 371 | print ("Unpacking tarball: " + pkg + " to " + outputdir) |
305 372 | cmd = ("tar -xf " + pkg + " -C " + outputdir).split() |
306 373 | print(cmd) |
307 374 | r = ShellRunner() |
308 375 | output = r.runshell(cmd, default_timeout, cwd=os.getcwd()) |
309 376 | installpath = outputdir + "/" + os.path.basename(pkg).replace(".tar.xz","") |
310 377 | return installpath |
311 378 | |
312 379 | def get_casatestutils_exec_path(pkg_dir): |
380 + | # Since runtest is no longer part of casatestutils, this may be removed. |
313 381 | for currentpath, folders, files in os.walk(pkg_dir): |
314 382 | for file in files: |
315 383 | #print(">>>" + os.path.join(currentpath, file)) |
316 384 | if currentpath.endswith('casatestutils') and file == 'runtest.py': |
317 385 | return(os.path.join(currentpath, file)) |
386 + | return "/dev/null/" |
318 387 | |
319 388 | def unpack_pkg(pkg, work_dir, outputdir): |
320 389 | if not os.path.exists(outputdir): |
321 390 | os.makedirs(outputdir) |
322 391 | |
323 392 | if platform.system() == "Linux": |
324 393 | installpath = unpack_tarball(pkg, outputdir) |
325 394 | print ("Package root: " + installpath) |
326 395 | exec_path = installpath + "/bin" |
327 396 | elif platform.system() == "Darwin": |
330 399 | exec_path = installpath + "/Contents/MacOS" |
331 400 | else: |
332 401 | raise Exception("Unknown operating system") |
333 402 | if exec_path is None: |
334 403 | raise Exception ("Couldn't find casa executable path") |
335 404 | casatestutils_exec_path = get_casatestutils_exec_path(installpath) |
336 405 | if casatestutils_exec_path == None: |
337 406 | raise Exception("Couldn't find casatestutils") |
338 407 | return exec_path, casatestutils_exec_path |
339 408 | |
340 - | |
341 - | ######################################################################################################################## |
342 - | ###################################### Imports / Constants ####################################### |
343 - | ######################################################################################################################## |
344 - | # Python Version |
345 - | PYVER = str(sys.version_info[0]) + "." + str(sys.version_info[1]) |
346 - | |
347 - | # mem mode variables |
348 - | HAVE_MEMTEST=True |
349 - | MEM = 0 |
350 - | try: |
351 - | import memTest |
352 - | except ImportError: |
353 - | HAVE_MEMTEST = False |
354 - | |
355 - | # cov mode variables |
356 - | HAVE_COVTEST=True |
357 - | COV = 0 |
358 - | try: |
359 - | import coverage |
360 - | except ImportError: |
361 - | HAVE_COVTEST = False |
362 - | |
363 - | # pybot mode variables |
364 - | HAVE_ROBOT = True |
365 - | USE_PYBOT = 0 |
366 - | try: |
367 - | import robot |
368 - | except ImportError: |
369 - | HAVE_ROBOT = False |
370 - | |
371 - | #### PYTEST IMPORT |
372 - | HAVE_PYTEST = True |
373 - | try: |
374 - | import pytest |
375 - | except ImportError: |
376 - | HAVE_PYTEST = False |
377 - | |
378 - | #### NOSE IMPORT |
379 - | HAVE_NOSE = True |
380 - | try: |
381 - | import nose |
382 - | except ImportError: |
383 - | HAVE_NOSE = False |
384 - | |
385 - | IS_CASA6 = False |
386 - | CASA6 = False |
387 - | HAVE_CASA6 = False |
388 - | |
389 - | # JIRA BRANCH TO CHECKOUT |
390 - | JIRA_BRANCH = None |
391 - | |
392 - | try: |
393 - | from __main__ import default |
394 - | from tasks import * |
395 - | from taskinit import * |
396 - | except ImportError: |
397 - | CASA6 = True |
398 - | IS_CASA6 = True |
399 - | HAVE_CASA6 = True |
400 - | |
401 - | # Use Nose attribute Functionality |
402 - | RUN_SUBTEST = False |
403 - | |
404 - | # Dry run of Tests |
405 - | DRY_RUN = False |
406 - | |
407 - | # Define which tests to run |
408 - | whichtests = 0 |
409 - | |
410 409 | ######################################################################################################################## |
411 410 | ############################################## Run ############################################### |
412 411 | ######################################################################################################################## |
413 412 | |
414 413 | def run(testnames): |
415 414 | |
416 415 | if IS_CASA6: |
417 416 | #sys.path.append(os.path.abspath(os.path.basename(__file__))) |
418 417 | if HAVE_PYTEST: |
419 418 | cwd = os.getcwd() + "/" |
450 449 | # Add 'junit_family=xunit1' to your pytest.ini file to keep the current format in future versions of pytest and silence this warning. |
451 450 | # _issue_warning_captured(deprecated.JUNIT_XML_DEFAULT_FAMILY, config.hook, 2) |
452 451 | ######### |
453 452 | cmd = ["--junitxml={}".format(xmlfile)] + ["-s"] + cmd |
454 453 | |
455 454 | if len(os.listdir(workdir + "all/")) == 0: |
456 455 | print("No Tests to Run") |
457 456 | else: |
458 457 | print("Running Command: pytest {}".format(cmd)) |
459 458 | pytest.main(cmd) |
460 - | |
459 + | |
461 460 | else: |
462 461 | print("Tests: {}".format(testnames)) |
463 462 | gittest = True |
464 463 | |
465 464 | for testname in testnames: |
466 465 | cmd = [] |
467 - | |
466 + | |
468 467 | # Copy Test To nosedir Directory if in cwd |
469 468 | if testname.startswith("test"): |
470 469 | test = testname |
471 470 | # Check if specific tests are requested |
472 471 | if "[" and "]" in test: |
473 472 | testname = getname(test) |
474 473 | tests = gettests(test) |
475 474 | #print(testname) |
476 475 | #print(tests) |
477 476 | |
481 480 | teststring = tests[0] |
482 481 | elif len(tests) > 1: |
483 482 | print(tests) |
484 483 | teststring = " or ".join(tests) |
485 484 | #workdir = os.getcwd() +"/workdir/nosedir/{}/".format(testname) |
486 485 | #os.makedirs(workdir) |
487 486 | #cmd = [ workdir ] |
488 487 | #cmd = [ ".".join([testname,'py']), "-k {}".format(teststring)] |
489 488 | cmd = ["-k {}".format(teststring)] + cmd |
490 489 | test = testname |
491 - | |
490 + | |
492 491 | # Set up Test Working Directory |
493 492 | if not os.path.exists(workdir + "{}/".format(test if not test.endswith(".py") else test[:-3])): |
494 493 | print("Setting Working Directory: {}".format(workdir + "{}/".format(test if not test.endswith(".py") else test[:-3]))) |
495 494 | os.makedirs(workdir + "{}/".format(test if not test.endswith(".py") else test[:-3])) |
496 495 | cmd = [ workdir + "{}/".format(test if not test.endswith(".py") else test[:-3]) ] + cmd |
497 - | |
498 496 | |
499 - | |
500 497 | # Check to see if tests need to be pulled from git. Only needs to be done once |
501 498 | if not test.endswith(".py") and gittest == True: |
502 499 | if JIRA_BRANCH is not None: |
503 500 | git_fetch_casa_tests_branch(workpath + 'casa6/', JIRA_BRANCH) |
504 501 | os.makedirs(workdir + "tests/") |
505 502 | gather_all_tests(workpath +'casa6/', workdir + "tests/") |
506 503 | gittest = False |
507 504 | |
508 505 | else: |
509 506 | print("Fetching Tests From Git Main Since No Local Test is Given") |
510 507 | git_fetch_casa_tests( workpath + 'casa6/') |
511 508 | os.makedirs(workdir + "tests/") |
512 509 | gather_all_tests(workpath +'casa6/', workdir + "tests/") |
513 510 | gittest = False |
514 - | |
511 + | |
515 512 | if test.endswith(".py"): |
516 513 | try: |
517 514 | print("Copying: {} to {}".format(test, workdir + "{}/".format(test if not test.endswith(".py") else test[:-3]))) |
518 515 | shutil.copy2(test, workdir + "{}/".format(test if not test.endswith(".py") else test[:-3])) |
519 516 | except: |
520 517 | traceback.print_exc() |
521 518 | else: |
522 519 | try: |
523 520 | print("Copying: {} to {}".format(workdir + "tests/",test), workdir + "{}/".format(test if not test.endswith(".py") else test[:-3])) |
524 521 | shutil.copy2("{}{}.py".format(workdir + "tests/",test), workdir + "{}/".format(test if not test.endswith(".py") else test[:-3])) |
525 522 | except: |
526 523 | traceback.print_exc() |
527 524 | |
528 525 | # https://docs.pytest.org/en/stable/usage.html |
529 526 | if verbose: |
530 - | cmd = ["--verbose"] + ["--tb=short"] + cmd |
527 + | cmd = ["--verbose"] + ["--tb=long"] + cmd |
531 528 | elif not verbose: |
532 - | cmd = ["-ra"] + ["--tb=line"] + cmd |
529 + | cmd = ["-ra"] + ["--tb=short"] + cmd |
530 + | #cmd = ["-ra"] + ["--tb=long"] + cmd |
533 531 | |
534 532 | if DRY_RUN: |
535 533 | cmd = ["--collect-only"] + cmd |
536 534 | |
537 535 | if not os.path.isdir(workpath + '/xml/{}/'.format(test if not test.endswith(".py") else test[:-3])): |
538 536 | os.makedirs(workpath + '/xml/{}/'.format(test if not test.endswith(".py") else test[:-3])) |
539 537 | xmlfile = workpath + 'xml/{}/nose.xml'.format(test if not test.endswith(".py") else test[:-3]) |
540 538 | |
541 539 | ######### |
542 540 | #============================================================ warnings summary ===================================================== |
553 551 | print("No Tests to Run") |
554 552 | sys.exit() |
555 553 | else: |
556 554 | |
557 555 | myworkdir = os.getcwd() |
558 556 | os.chdir("{}".format(workdir + "{}/".format(test if not test.endswith(".py") else test[:-3]))) |
559 557 | print("Test Directory: {}".format(os.getcwd())) |
560 558 | print("Running Command: pytest {}".format(cmd)) |
561 559 | pytest.main(cmd) |
562 560 | os.chdir(myworkdir) |
563 - | |
561 + | |
564 562 | ################################################## |
565 563 | ########## Real Path ########## |
566 564 | ################################################## |
567 565 | # Copy Test To nosedir Directory assuming it's in another location |
568 566 | elif testname.startswith("/"): |
569 567 | testpath = testname.split("[")[0] |
570 568 | cmd = [] |
571 569 | dirname = testname.split("/")[-1] |
572 570 | test = dirname |
573 571 | if "[" and "]" in test: |
594 592 | |
595 593 | # Set up Test Working Directory |
596 594 | if not os.path.exists(workdir + "{}/".format(dirname)): |
597 595 | print("Setting Working Directory: {}".format(workdir + "{}/".format(dirname))) |
598 596 | os.makedirs(workdir + "{}/".format(dirname)) |
599 597 | cmd = [ workdir + "{}/".format(dirname) ] + cmd |
600 598 | try: |
601 599 | shutil.copy2(testpath, workdir + "{}/".format(dirname)) |
602 600 | except: |
603 601 | traceback.print_exc() |
604 - | |
602 + | |
605 603 | if verbose: |
606 - | cmd = ["--verbose"] + ["--tb=short"] + cmd |
604 + | cmd = ["--verbose"] + ["--tb=long"] + cmd |
607 605 | elif not verbose: |
608 - | cmd = ["-ra"] + ["--tb=line"] + cmd |
606 + | cmd = ["-ra"] + ["--tb=short"] + cmd |
607 + | #cmd = ["-ra"] + ["--tb=long"] + cmd |
609 608 | |
610 609 | |
611 610 | if DRY_RUN: |
612 611 | cmd = ["--collect-only"] + cmd |
613 612 | |
614 613 | if not os.path.isdir(workpath + '/xml/{}/'.format(dirname)): |
615 614 | os.makedirs(workpath + '/xml/{}/'.format(dirname)) |
616 615 | xmlfile = workpath + 'xml/{}/nose.xml'.format(dirname) |
617 616 | |
618 617 | ######### |
629 628 | if len(os.listdir(workpath)) < 1: # If only the XML dir was created |
630 629 | print("No Tests to Run") |
631 630 | sys.exit() |
632 631 | else: |
633 632 | myworkdir = os.getcwd() |
634 633 | os.chdir(workdir + "{}/".format(dirname)) |
635 634 | print("Test Directory: {}".format(os.getcwd())) |
636 635 | print("Running Command: pytest {}".format(cmd)) |
637 636 | pytest.main(cmd) |
638 637 | os.chdir(myworkdir) |
639 - | |
640 - | |
641 638 | os.chdir(cwd) |
642 639 | |
643 - | if not IS_CASA6: # If in CASA5 |
644 - | if HAVE_NOSE: |
645 - | # Global variable used by regression framework to determine pass/failure status |
646 - | global regstate |
647 - | regstate = False |
648 - | |
649 - | listtests = testnames |
650 - | if listtests == '--Help' or listtests == []: |
651 - | usage() |
652 - | sys.exit() |
653 - | |
654 - | if listtests == '--list': |
655 - | list_tests() |
656 - | sys.exit() |
657 - | |
658 - | if listtests == 'all': |
659 - | whichtests = 0 |
660 - | # Get the full list of tests from file |
661 - | listtests = readfile(LISTofTESTS) |
662 - | if listtests == []: |
663 - | raise Exception('List of tests "{}" is empty or does not exist'.format(LISTofTESTS)) |
664 - | |
665 - | elif (type(testnames) != type([])): |
666 - | if (os.path.isfile(testnames)): |
667 - | # How to prevent it from opening a real test??? |
668 - | whichtests = 1 |
669 - | listtests = readfile(testnames) |
670 - | if listtests == []: |
671 - | raise Exception('List of tests is empty') |
672 - | else: |
673 - | raise Exception('List of tests does not exist') |
674 - | |
675 - | else: |
676 - | # run specific tests |
677 - | whichtests = 1 |
678 - | |
679 - | |
680 - | # Directories |
681 - | PWD = os.getcwd() |
682 - | WDIR = PWD+'/nosedir/' |
683 - | |
684 - | # Create a working directory |
685 - | workdir = WDIR |
686 - | print("Creating work directory: {}".format(workdir)) |
687 - | if os.access(workdir, os.F_OK) is False: |
688 - | os.makedirs(workdir) |
689 - | else: |
690 - | shutil.rmtree(workdir) |
691 - | os.makedirs(workdir) |
692 - | |
693 - | # Move to working dir |
694 - | os.chdir(workdir) |
695 - | |
696 - | # Create a directory for nose's xml files |
697 - | xmldir = WDIR+'xml/' |
698 - | if os.access(xmldir, os.F_OK) is False: |
699 - | os.makedirs(xmldir) |
700 - | else: |
701 - | shutil.rmtree(xmldir) |
702 - | os.makedirs(xmldir) |
703 - | |
704 - | print("Starting unit tests for {}: ".format(listtests)) |
705 - | |
706 - | # ASSEMBLE and RUN the TESTS |
707 - | if not whichtests: |
708 - | '''Run all tests''' |
709 - | list = [] |
710 - | testlist_to_execute= [] |
711 - | # awells CAS-10844 Fix |
712 - | suiteList = [] |
713 - | for f in listtests: |
714 - | suite = unittest.TestSuite() |
715 - | try: |
716 - | tests = UnitTest(f).getUnitTest() |
717 - | if RUN_SUBTEST: |
718 - | testlist_to_execute = testlist_to_execute + getsubtests(f,tests) |
719 - | for test in tests: |
720 - | suite.addTest(test) |
721 - | suiteList.append(suite) |
722 - | except: |
723 - | traceback.print_exc() |
724 - | list = suiteList |
725 - | |
726 - | elif (whichtests == 1): |
727 - | '''Run specific tests''' |
728 - | list = [] |
729 - | testlist_to_execute= [] |
730 - | for f in listtests: |
731 - | if not haslist(f): |
732 - | testcases = UnitTest(f).getUnitTest() |
733 - | list = list+testcases |
734 - | |
735 - | if RUN_SUBTEST: |
736 - | testlist_to_execute = testlist_to_execute + getsubtests(f,list) |
737 - | else: |
738 - | ff = getname(f) |
739 - | tests = gettests(f) |
740 - | # allow splitting of large test groups into smaller chunks |
741 - | # large long running test groups make parallel test scheduling |
742 - | # a bit more complicated so splitting them to smaller groups |
743 - | # helps |
744 - | # syntax: [testsplit:chunk_index-number_of_chunks] |
745 - | if len(tests) == 1 and tests[0].startswith('testsplit:'): |
746 - | import math |
747 - | testcases = UnitTest(ff).getUnitTest() |
748 - | chk, nchk = map(int, tests[0].split(':')[1].split('-')) |
749 - | if chk > nchk or chk < 1: |
750 - | raise ValueError('testsplit chunk must be 1 <= nchunks') |
751 - | nchk = min(len(testcases), nchk) |
752 - | chksz = int(math.ceil(len(testcases) / float(nchk))) |
753 - | offset = (chk - 1) * chksz |
754 - | print("running tests {} to {}".format(offset, min(offset + chksz, len(testcases)))) |
755 - | testcases = testcases[offset:offset + chksz] |
756 - | else: |
757 - | testcases = UnitTest(ff).getUnitTest(tests) |
758 - | list = list+testcases |
759 - | |
760 - | if RUN_SUBTEST: |
761 - | |
762 - | |
763 - | if len(testlist_to_execute) == 0: |
764 - | raise ValueError("Cannot Find Tests with Attribute:'{}'".format(ATTR_VAL)) |
765 - | if not whichtests: |
766 - | for i in range(0,len(list)): |
767 - | tmp = [] |
768 - | for item in list[i]: |
769 - | if [item._testMethodName,item.__module__] in testlist_to_execute: |
770 - | tmp.append(item) |
771 - | list[i] = unittest.TestSuite(tmp) |
772 - | else: |
773 - | tmp = [] |
774 - | for item in list: |
775 - | if [item._testMethodName,item.__module__] in testlist_to_execute: |
776 - | tmp.append(item) |
777 - | list = tmp |
778 - | |
779 - | if (len(list) == 0): |
780 - | os.chdir(PWD) |
781 - | raise Exception('ERROR: There are no valid tests to run') |
782 - | |
783 - | |
784 - | # Run all tests and create a XML report |
785 - | xmlfile = xmldir+'nose.xml' |
640 + | if not HAVE_PYTEST: |
641 + | print("Missing Pytest") |
786 642 | |
787 - | try: |
788 - | if (HAVE_MEMTEST and MEM): |
789 - | regstate = nose.run(argv=[sys.argv[0],"-d","-s","--with-memtest","--verbosity=2", |
790 - | "--memtest-file="+xmlfile], suite=list, addplugins=[memTest.MemTest()]) |
791 - | else: |
792 - | regstate = nose.run(argv=[sys.argv[0],"-d","-s","--with-xunit","--verbosity=2", |
793 - | "--xunit-file="+xmlfile], suite=list) |
794 - | |
795 - | os.chdir(PWD) |
796 - | except: |
797 - | print("Failed to run one or more tests") |
798 - | traceback.print_exc() |
799 - | else: |
800 - | os.chdir(PWD) |
801 643 | |
802 644 | ######################################################################################################################## |
803 645 | ####################################### Run Bamboo Option ######################################## |
804 646 | ######################################################################################################################## |
805 647 | |
806 648 | def run_bamboo(pkg, work_dir, branch = None, test_group = None, test_list= None, test_paths = [], test_config_path=None, ncores=2, verbosity=False, pmode=None): |
807 649 | |
808 650 | if test_list is not None: |
809 651 | test_list = [x.strip() for x in test_list.split(',')] |
810 652 | if args.test_group is not None: |
811 653 | test_group = [x.strip() for x in test_group.split(',')] |
654 + | |
812 655 | # Unpack the distribution |
813 656 | print ("Test list: " + str (test_list)) |
814 657 | print ("Test group: " + str (test_group)) |
815 658 | if pkg is None: |
816 659 | raise Exception("Missing pkg") |
817 660 | if work_dir is None: |
818 661 | raise Exception("Missing work_dir") |
819 - | #exec_path, casatestutils_exec_path = unpack_pkg(pkg, work_dir, work_dir + "/pkg") |
820 - | exec_path = "work/pkg/casa-6.3.0-11/bin" |
821 - | casatestutils_exec_path = "work/pkg/casa-6.3.0-11/lib/py/lib/python3.6/site-packages/casatestutils/runtest.py" |
822 - | #exec_path = "/tmp/work/pkg/CASA.app/Contents/MacOS" |
662 + | exec_path, casatestutils_exec_path = unpack_pkg(pkg, work_dir, work_dir + "/pkg") |
663 + | |
823 664 | print("Executable path: " + exec_path) |
824 665 | print("casatestutils path: " + casatestutils_exec_path) |
825 666 | |
826 667 | # Start Xvfb on Linux |
827 668 | xvfb = xvfb_helper.XvfbHelper() |
828 669 | if sys.platform != "darwin": |
829 670 | xvfb.start_virtual_frame_buffer() |
830 671 | |
831 672 | if args.branch == None: |
832 673 | branch = "master" |
833 674 | |
834 675 | # Clone a default set of repositories to if test paths are not provided from command line |
835 - | #if len(test_paths) == 0 : |
836 - | # test_paths = fetch_tests(str(work_dir), branch) |
837 - | test_paths = ['work/casasources//casampi/src/casampi/tests', 'work/casasources//casaplotms/tests/plotms', 'work/casasources//almatasks/tests/tasks', 'work/casasources//casa6/casatests/regression/', 'work/casasources//casa6/casatests/stakeholder/', 'work/casasources//casa6/casatasks/tests/', 'work/casasources//casa6/casatools/tests/', 'work/casasources//casaviewer/tests/tasks'] |
676 + | if len(test_paths) == 0 : |
677 + | test_paths = fetch_tests(str(work_dir), branch) |
838 678 | |
839 - | |
840 679 | test_config = None |
841 680 | if test_config_path == None: |
842 681 | test_config_path = work_dir + "/casasources/casa6/casatestutils/casatestutils/component_to_test_map.json" |
843 682 | # Read the JSON configuration |
844 683 | print ("Reading config from: " + test_config_path) |
845 684 | with open(test_config_path ) as f: |
846 685 | test_config = json.load(f) |
847 686 | |
848 687 | # Get the actual tests as list |
849 688 | test_config_elems = test_config['testlist'] |
850 689 | |
851 690 | #print(test_config_elems) |
852 - | print("TPS: ", test_paths) |
691 + | print("Test Paths: ", test_paths) |
853 692 | |
854 693 | # Match the test names provided in the JSON file to the actual test locations. |
855 694 | tests_to_run = [] |
856 695 | for x in test_config_elems: |
857 696 | for dir in test_paths: |
858 697 | for currentpath, folders, files in os.walk(dir): |
859 698 | for file in files: |
860 699 | if file == (x['testScript']+".py"): |
861 700 | test_location = os.path.join(currentpath, file) |
862 701 | if verbosity: |
872 711 | |
873 712 | tests_to_run.append(casa_test(x['testScript'], |
874 713 | test_location, |
875 714 | x['testGroup'], |
876 715 | x['testType'], |
877 716 | x['maintainer'], |
878 717 | x['maintainerEmail'], |
879 718 | tuple(opts), |
880 719 | x['comment'], |
881 720 | timeout)) |
882 - | #print(tests_to_run) |
721 + | |
883 722 | # Filter tests by test list |
884 723 | if test_list is not None and len(test_list)>0: |
885 724 | print ("Test list provided. Filtering tests.") |
886 725 | tmp_tests_to_run = [] |
887 726 | for test in test_list: |
888 727 | found = False |
889 728 | for t1 in tests_to_run: |
890 729 | if test == t1.name: |
891 730 | tmp_tests_to_run.append(t1) |
892 731 | found = True |
903 742 | if file == test + ".py": |
904 743 | test_location = os.path.join(currentpath, file) |
905 744 | print("Found: " + test_location) |
906 745 | print("No JSON configuration found. Test will be added to execution list without options.") |
907 746 | tmp_tests_to_run.append(casa_test(test, test_location, "","","","","","")) |
908 747 | found = True |
909 748 | if not found: |
910 749 | raise Exception("Couldn't locate test: " + test) |
911 750 | tests_to_run = tmp_tests_to_run |
912 751 | |
913 - | #print(tests_to_run) |
914 752 | # Filter by Jira components |
915 753 | if test_group is not None and len(test_group)>0: |
916 754 | print("Jira component list provided. Filtering tests.") |
917 755 | tmp_tests_to_run = [] |
918 756 | #print(test_group) |
919 757 | for jira_component in test_group: |
920 758 | found = False |
921 759 | for t1 in tests_to_run: |
922 760 | if jira_component.lower() in [x.lower() for x in t1.test_group]: |
923 761 | tmp_tests_to_run.append(t1) |
941 779 | str(t.timeout) + " : " + |
942 780 | t.comment) |
943 781 | else: |
944 782 | print(t.name) |
945 783 | |
946 784 | # Run tests |
947 785 | print("") |
948 786 | for test in tests_to_run: |
949 787 | r = ShellRunner() |
950 788 | xunit = Xunit() |
951 - | casaopts = " --nogui --nologger --log2term" |
952 789 | |
953 790 | #pmodes: pmodes = ['serial','parallel','both'] |
954 791 | # Skip MPI on Darwin for now |
955 792 | if "mpi" in test.options and sys.platform != "darwin" and ( pmode == 'parallel' or pmode == 'both'): |
956 793 | print("Running test: {} in MPI mode".format(test.name)) |
957 794 | casa_exe = exec_path + "/mpicasa" |
958 795 | casaopts = "-n " + str(ncores) + " " + exec_path + "/casa" + " --nogui --nologger --log2term --agg " |
959 796 | assert (test != None) |
960 - | #cmd = (casa_exe + " " + casaopts + " -c " + casatestutils_exec_path + " " + test.path).split() |
961 797 | cmd = (casa_exe + " " + casaopts + " -c " + test.path).split() |
962 798 | cwd = work_dir + "/" + test.name |
963 799 | if pmode == 'both': |
964 800 | cwd = work_dir + "/" + test.name + '_mpi' |
965 801 | |
802 + | print("Running cmd " + str(cmd) + "in " + cwd) |
966 803 | if not os.path.exists(cwd): |
967 804 | os.makedirs(cwd) |
968 - | print("Running cmd " + str(cmd)) |
969 - | print("in " + cwd) |
970 805 | starttime = datetime.datetime.now() |
971 - | |
972 806 | output = r.runshell(cmd, test.timeout,cwd) |
973 - | |
974 807 | endtime = datetime.datetime.now() |
975 808 | runtime = endtime - starttime |
976 809 | xunit.append_result(test.name, str(runtime), len(output), output) |
977 810 | print("") |
811 + | |
978 812 | if pmode == 'both': |
979 813 | if "casampi" in test.name: |
980 814 | continue |
981 815 | print("Running test: {} in Serial mode".format(test.name)) |
982 816 | casaopts = " --nogui --nologger --log2term" |
983 817 | casa_exe = exec_path + "/casa" |
984 818 | assert (test != None) |
985 - | #cmd = (casa_exe + " " + casaopts + " -c " + casatestutils_exec_path + " " + test.path).split() |
986 819 | cmd = (casa_exe + " " + casaopts + " -c " + test.path).split() |
987 820 | cwd = work_dir + "/" + test.name |
821 + | |
822 + | print("Running cmd " + str(cmd) + "in " + cwd) |
988 823 | if not os.path.exists(cwd): |
989 824 | os.makedirs(cwd) |
990 - | print("Running cmd " + str(cmd)) |
991 - | print("in " + cwd) |
992 825 | starttime = datetime.datetime.now() |
993 - | |
994 826 | output = r.runshell(cmd, test.timeout,cwd) |
995 - | |
996 827 | endtime = datetime.datetime.now() |
997 828 | runtime = endtime - starttime |
998 829 | xunit.append_result(test.name, str(runtime), len(output), output) |
999 830 | print("") |
1000 831 | |
1001 - | #elif "mpi" not in test.options and ( pmode == 'serial'): |
832 + | elif "mpi" not in test.options and sys.platform != "darwin" and ( pmode == 'parallel' or pmode == 'both'): |
833 + | # Special Case when you have tests in the HPC / parallel list but no mpi test option |
834 + | print("Running test: {} in Serial mode".format(test.name)) |
835 + | casaopts = " --nogui --nologger --log2term" |
836 + | casa_exe = exec_path + "/casa" |
837 + | assert (test != None) |
838 + | cmd = (casa_exe + " " + casaopts + " -c " + test.path).split() |
839 + | cwd = work_dir + "/" + test.name |
840 + | |
841 + | print("Running cmd " + str(cmd) + "in " + cwd) |
842 + | if not os.path.exists(cwd): |
843 + | os.makedirs(cwd) |
844 + | starttime = datetime.datetime.now() |
845 + | output = r.runshell(cmd, test.timeout,cwd) |
846 + | endtime = datetime.datetime.now() |
847 + | runtime = endtime - starttime |
848 + | xunit.append_result(test.name, str(runtime), len(output), output) |
849 + | print("") |
850 + | |
1002 851 | elif pmode == 'serial': |
1003 852 | if "casampi" in test.name: |
1004 853 | continue |
1005 854 | print("Running test: {} in Serial mode".format(test.name)) |
855 + | casaopts = " --nogui --nologger --log2term" |
1006 856 | casa_exe = exec_path + "/casa" |
1007 857 | assert (test != None) |
1008 - | #cmd = (casa_exe + " " + casaopts + " -c " + casatestutils_exec_path + " " + test.path).split() |
1009 858 | cmd = (casa_exe + " " + casaopts + " -c " + test.path).split() |
1010 859 | cwd = work_dir + "/" + test.name |
860 + | |
861 + | print("Running cmd " + str(cmd) + "in " + cwd) |
1011 862 | if not os.path.exists(cwd): |
1012 863 | os.makedirs(cwd) |
1013 - | print("Running cmd " + str(cmd)) |
1014 - | print("in " + cwd) |
1015 864 | starttime = datetime.datetime.now() |
1016 - | |
1017 865 | output = r.runshell(cmd, test.timeout,cwd) |
1018 - | |
1019 866 | endtime = datetime.datetime.now() |
1020 867 | runtime = endtime - starttime |
1021 868 | xunit.append_result(test.name, str(runtime), len(output), output) |
1022 869 | print("") |
1023 870 | |
1024 871 | xunit.generateXml("suite") |
1025 872 | |
1026 873 | # Close Xvfb on exit |
1027 874 | import atexit |
1028 875 | register | .
1029 876 | def goodbye(): |
1030 877 | print("Stopping Xvfb.") |
1031 878 | if sys.platform != "darwin": |
1032 879 | xvfb.signal_stop_virtual_frame_buffer() |
1033 880 | print("Xvfb stopped.") |
1034 881 | |
1035 882 | ######################################################################################################################## |
1036 883 | ######################################## Main-Start-Up ########################################### |
1037 884 | ######################################################################################################################## |
1038 885 | |
886 + | |
1039 887 | if __name__ == "__main__": |
1040 888 | |
1041 - | # mem mode variables |
1042 889 | print("HAVE_MEMTEST: {}".format(HAVE_MEMTEST)) |
1043 890 | print("HAVE_COVTEST: {}".format(HAVE_COVTEST)) |
1044 891 | print("HAVE_ROBOT: {}".format(HAVE_ROBOT)) |
1045 892 | print("HAVE_PYTEST: {}".format(HAVE_PYTEST)) |
1046 893 | print("HAVE_NOSE: {}".format(HAVE_NOSE)) |
1047 894 | print("HAVE_CASA6: {}".format(HAVE_CASA6)) |
1048 895 | print("") |
1049 896 | |
1050 - | verbose = False |
897 + | |
1051 898 | |
1052 - | #original_datapath = casa['dirs']['data'] |
1053 899 | # List of tests to run |
1054 900 | testnames = [] |
1055 - | |
901 + | |
1056 902 | parser = argparse.ArgumentParser(allow_abbrev=False) |
1057 903 | parser.add_argument("-a", "--all", action='store_true',help='run all tests defined in trunk/gcwrap/python/scripts/tests/uTest_list.json.') |
1058 904 | parser.add_argument("-i", "--list",action='store_true',help='print the list of tests & tags defined in ') |
1059 905 | parser.add_argument("-v", "--verbose",action='store_true',help="Verbose Test Execution") |
1060 906 | parser.add_argument("-x", "--dry-run",action='store_true',help="dry run Test Execution") |
1061 907 | parser.add_argument("-s", "--classes",nargs='+',metavar='test',help='print the classes from a test script') |
1062 908 | parser.add_argument("-f", "--file",nargs='?', type=argparse.FileType('r'),help='run the tests defined in an ASCII file <list>; one test per line') |
1063 - | #parser.add_argument("-m", "--mem",action='store_true',help='show the memory used by the tests and the number of files left open.') |
1064 909 | |
1065 910 | # Component Arguments |
1066 - | |
1067 911 | parser.add_argument("-e","--mapfile", nargs='?', type=argparse.FileType('r'), help='Component to test map file', required=False) |
1068 912 | parser.add_argument("-b","--branch", help='JIRA Branch for test repository checkouts', required=False) |
1069 913 | |
1070 914 | # casa-build-utils Arguments |
1071 915 | parser.add_argument('-p','--pkg', help='Tarball or dmg', required=False) |
1072 916 | parser.add_argument('-w','--work_dir', help='Working directory.', required=False) |
1073 917 | parser.add_argument('-n','--ncores', help='Number of cores for MPI tests', default=2) |
1074 918 | parser.add_argument('-t','--test_paths', help='A comma separated list of paths containing tests.', required=False) |
1075 919 | parser.add_argument('-l','--test_list', help='Filter tests by a comma separated list of tests', required=False) |
1076 920 | parser.add_argument('-c','--test_config', help='Test configuration file', required=False) |
1077 921 | parser.add_argument('-j','--test_group', help='Filter tests by a comma separated list of components', required=False) |
1078 - | # parser.add_argument('-b','--branch', help='Branch for test repository checkouts', required=False) ## Duplicate |
1079 - | |
1080 922 | parser.add_argument('-m','--pmode', help='Parallelization mode: serial, parallel, both', required=False) |
1081 - | |
1082 923 | parser.add_argument('--bamboo', help='Set Bamboo Flag to True',default=False,action='store_true', required=False) |
1083 924 | |
1084 925 | if not IS_CASA6: |
1085 926 | if "-c" in sys.argv: |
1086 927 | i = sys.argv.index("-c") |
1087 928 | args, unknownArgs = parser.parse_known_args(sys.argv[i+2:]) |
1088 929 | else: |
1089 930 | args, unknownArgs = parser.parse_known_args() |
1090 931 | |
1091 932 | print(args) |
1092 933 | print("") |
1093 - | #args=parser.parse_args() |
1094 934 | |
1095 935 | print("Operating system: " + platform.system()) |
1096 936 | print("") |
1097 - | # print(unknownArgs) |
1098 937 | |
1099 938 | if args.test_group is not None: |
1100 939 | components = args.test_group |
1101 940 | components = components.split(",") |
1102 941 | print("Testing Components" + str(components)) |
1103 942 | print("") |
1104 943 | |
1105 - | # |
1106 - | if args.mapfile is not None: |
1107 - | component_to_test_map = json.load(args.mapfile) |
1108 - | else: |
1109 - | import casatestutils as _; |
1110 - | with open("{}/{}".format(_.__path__[0], "component_to_test_map.json")) as ctt: |
1111 - | component_to_test_map = json.load(ctt) |
1112 - | |
1113 - | for c in components: |
1114 - | _isComponent = False |
1115 - | component = c.strip() |
1116 - | for myDict in component_to_test_map["testlist"]: |
1117 - | #print(component, myDict["testGroup"]) |
1118 - | if component in myDict["testGroup"]: |
1119 - | _isComponent = True |
1120 - | testnames.append(myDict["testScript"]) |
1121 - | if not _isComponent: |
1122 - | print("No Tests for Component: {}".format(component)) |
944 + | if not args.bamboo: |
945 + | if args.mapfile is not None: |
946 + | component_to_test_map = json.load(args.mapfile) |
947 + | else: |
948 + | try: |
949 + | import casatestutils as _; |
950 + | with open("{}/{}".format(_.__path__[0], "component_to_test_map.json")) as ctt: |
951 + | component_to_test_map = json.load(ctt) |
952 + | except: |
953 + | print("No JSON file to Map") |
954 + | |
955 + | for c in components: |
956 + | _isComponent = False |
957 + | component = c.strip() |
958 + | for myDict in component_to_test_map["testlist"]: |
959 + | #print(component, myDict["testGroup"]) |
960 + | if component in myDict["testGroup"]: |
961 + | _isComponent = True |
962 + | testnames.append(myDict["testScript"]) |
963 + | if not _isComponent: |
964 + | print("No Tests for Component: {}".format(component)) |
1123 965 | |
1124 966 | if args.verbose: |
1125 967 | verbose = True |
1126 968 | if args.list: |
1127 969 | list_tests() |
1128 970 | sys.exit() |
971 + | |
1129 972 | ## Dry Run |
1130 - | DRY_RUN = False |
1131 973 | if args.dry_run: |
1132 974 | DRY_RUN = True |
1133 975 | |
1134 976 | ## RUN ALL |
1135 - | RUN_ALL = False |
1136 977 | if args.all: |
1137 978 | RUN_ALL = True |
1138 979 | testnames = ["all"] |
1139 980 | |
1140 981 | if args.classes is not None: |
1141 982 | print(args.classes) |
1142 983 | getclasses(args.classes) |
1143 984 | os._exit(0) |
1144 - | |
1145 985 | |
1146 986 | if args.file is not None: |
1147 987 | #logger.info('Reading Test List from %s: ', args.file) |
1148 988 | for line in args.file: |
1149 989 | try: |
1150 990 | #logger.debug("Adding Test %s from file %s",re.sub(r'[\n\r]+', '',line),args.file) |
1151 991 | testnames.append(re.sub(r'[\n\r]+', '',line)) |
1152 992 | except: |
1153 993 | raise Exception(" The list should contain one test per line.") |
1154 - | |
1155 - | #if args.mem: # run specific tests in mem mode |
1156 - | # #logger.info('Setting Mem Mode') |
1157 - | # MEM = 1 |
1158 - | |
1159 994 | |
1160 995 | if args.branch is not None: |
1161 996 | JIRA_BRANCH = args.branch |
1162 997 | |
1163 998 | test_paths = [] |
1164 999 | if args.test_paths is not None: |
1165 1000 | test_paths = [x.strip() for x in args.test_paths.split(',')] |
1166 1001 | |
1167 1002 | for arg in unknownArgs: |
1003 + | |
1168 1004 | if arg.startswith(("-", "--")): |
1169 1005 | raise ValueError('unrecognized argument: %s'%(arg)) |
1170 1006 | sys.exit() |
1171 - | # |
1007 + | |
1172 1008 | else: |
1009 + | |
1173 1010 | if arg.startswith("test") and not RUN_ALL: |
1174 1011 | testnames.append(arg) |
1175 1012 | # local Path file |
1176 1013 | #elif arg.startswith("./") and arg.endswith(".py") and not RUN_ALL: |
1177 1014 | elif arg.startswith("./") and ".py" in arg and not RUN_ALL: |
1178 1015 | |
1179 - | #testnames.append(arg[2:]) |
1180 1016 | try: |
1181 1017 | real_path = os.path.realpath(arg[2:]) |
1182 - | #print("real_path: {}".format(real_path)) |
1183 1018 | testnames.append(real_path) |
1184 1019 | except: |
1185 1020 | traceback.print_exc() |
1186 1021 | |
1187 1022 | elif (arg.startswith("../") or arg.startswith("/")) and ".py" in arg and not RUN_ALL: |
1188 1023 | try: |
1189 1024 | real_path = os.path.realpath(arg) |
1190 - | #print("real_path: {}".format(real_path)) |
1191 1025 | testnames.append(real_path) |
1192 1026 | except: |
1193 1027 | traceback.print_exc() |
1194 1028 | |
1195 1029 | try: |
1196 - | |
1197 1030 | if args.bamboo: |
1031 + | from testrunner.shell_runner import ShellRunner |
1032 + | from testrunner import xvfb_helper |
1033 + | from testrunner.xunit import Xunit |
1198 1034 | if args.pkg: |
1199 1035 | print("Package: " + args.pkg) |
1200 1036 | print("Test configuration file: " + str(args.test_config)) |
1201 1037 | print("Number of cores: " + str(args.ncores)) |
1202 1038 | print("Workdir: " + str(args.work_dir)) |
1203 1039 | pmodes = ['serial','parallel','both'] |
1204 1040 | if args.pmode not in pmodes: |
1205 1041 | raise Exception("Invalid pmode: '{}'. Valid modes: '{}'".format(args.pmode ,str(pmodes))) |
1206 1042 | |
1207 1043 | run_bamboo(args.pkg, args.work_dir, args.branch, args.test_group, args.test_list, test_paths, args.test_config, args.ncores, args.verbose, args.pmode) |
1208 1044 | |
1209 1045 | else: |
1210 1046 | #If no tests are given, no subet tag or --all option |
1047 + | #print("Testnames: {}".format(testnames)) |
1048 + | if args.test_paths is not None: |
1049 + | tests = [] |
1050 + | test_paths = [x.strip() for x in args.test_paths.split(',')] |
1051 + | if len(testnames) == 0: |
1052 + | for test_path in test_paths: |
1053 + | for root, dirs, files in os.walk(test_path): |
1054 + | for file in files: |
1055 + | if file.endswith(".py") and file.startswith("test_"): |
1056 + | tests.append(os.path.join(root, file)) |
1057 + | else: |
1058 + | for test_path in test_paths: |
1059 + | #print(test_path) |
1060 + | for test in testnames: |
1061 + | if not test.endswith(".py"): |
1062 + | test = test + ".py" |
1063 + | #print(test) |
1064 + | for root, dirs, files in os.walk(test_path): |
1065 + | for file in files: |
1066 + | if file == test: |
1067 + | tests.append(os.path.join(root, file)) |
1068 + | testnames = tests |
1069 + | |
1211 1070 | print("Testnames: {}".format(testnames)) |
1071 + | #print("Test: {}".format(tests)) |
1072 + | #sys.exit() |
1212 1073 | if testnames == [] or len(testnames) == 0: |
1213 - | #TODO |
1214 1074 | print("List of tests is empty") |
1215 1075 | parser.print_help(sys.stderr) |
1216 1076 | sys.exit(1) |
1217 1077 | #sys.exit() |
1218 1078 | #raise Exception("List of tests is empty") |
1219 1079 | run(testnames) |
1220 1080 | except: |
1221 1081 | traceback.print_exc() |
1222 1082 | |
1223 1083 | |