Commits
Kristin Berry authored d4c84106ce1 Merge
507 507 | # The keys are the session names |
508 508 | # The values are a tuple containing the vislist and the caltables |
509 509 | sessiondict = collections.OrderedDict() |
510 510 | for i in range(len(session_names)): |
511 511 | sessiondict[session_names[i]] = \ |
512 512 | ([os.path.basename(visfile) for visfile in session_vislists[i]], \ |
513 513 | os.path.basename(caltable_file_list[i])) |
514 514 | |
515 515 | return sessiondict |
516 516 | |
517 - | def _do_if_auxiliary_products(self, oussid, output_dir, products_dir, vislist, imaging_products_only): |
517 + | def _do_if_auxiliary_products(self, oussid, output_dir, products_dir, vislist, imaging_products_only, pipeline_stats_file=None): |
518 518 | """ |
519 519 | Generate the auxiliary products |
520 520 | """ |
521 - | |
522 521 | if imaging_products_only: |
523 522 | contfile_name = 'cont.dat' |
524 523 | fluxfile_name = 'Undefined' |
525 524 | antposfile_name = 'Undefined' |
526 525 | else: |
527 526 | fluxfile_name = 'flux.csv' |
528 527 | antposfile_name = 'antennapos.csv' |
529 528 | contfile_name = 'cont.dat' |
530 529 | empty = True |
531 530 | |
565 564 | if timetracker_file_list: |
566 565 | empty = False |
567 566 | |
568 567 | # PIPE-1802: look for the selfcal/restore resources |
569 568 | selfcal_resources_list = [] |
570 569 | if hasattr(self.inputs.context, 'selfcal_resources') and isinstance(self.inputs.context.selfcal_resources, list): |
571 570 | selfcal_resources_list = self.inputs.context.selfcal_resources |
572 571 | if selfcal_resources_list: |
573 572 | empty = False |
574 573 | |
574 + | # PIPE-2094: check for the pipeline stats file |
575 + | if pipeline_stats_file and os.path.exists(pipeline_stats_file): |
576 + | empty = False |
577 + | |
575 578 | if empty: |
576 579 | return None |
577 580 | |
578 581 | # Define the name of the output tarfile |
579 582 | tarfilename = f'{oussid}.auxproducts.tgz' |
580 583 | LOG.info('Saving auxiliary data products in %s', tarfilename) |
581 584 | |
582 585 | # Open tarfile |
583 586 | with tarfile.open(os.path.join(products_dir, tarfilename), 'w:gz') as tar: |
584 587 | |
618 621 | LOG.info('Saving auxiliary data product %s in %s', os.path.basename(timetracker_file), tarfilename) |
619 622 | else: |
620 623 | LOG.info('Auxiliary data product timetracker json report file does not exist') |
621 624 | |
622 625 | # PIPE-1802: Save selfcal restore resources |
623 626 | for selfcal_resource in selfcal_resources_list: |
624 627 | if os.path.exists(selfcal_resource): |
625 628 | tar.add(selfcal_resource, arcname=selfcal_resource) |
626 629 | LOG.info('Saving auxiliary data product %s in %s', selfcal_resource, tarfilename) |
627 630 | |
631 + | # PIPE-2094: Save pipeline statistics file |
632 + | if pipeline_stats_file and os.path.exists(pipeline_stats_file): |
633 + | tar.add(pipeline_stats_file, arcname=pipeline_stats_file) |
634 + | LOG.info('Saving pipeline statistics file %s in %s', pipeline_stats_file, tarfilename) |
635 + | else: |
636 + | LOG.info("Pipeline statistics file does not exist.") |
628 637 | tar.close() |
629 638 | |
630 639 | return tarfilename |
631 640 | |
632 641 | def _make_pipe_manifest(self, context, oussid, stdfproducts, sessiondict, msvisdict, exportmses, calvisdict, |
633 642 | exportcalprods, calimages, calimages_fitskeywords, targetimages, targetimages_fitskeywords): |
634 643 | """ |
635 644 | Generate the manifest file |
636 645 | """ |
637 646 | |