Commits

Rui Xue authored da11964ccbb Merge 
Merge remote-tracking branch 'origin/main' into PIPE-1033-flagging-of-zeros-is-not-reported-correctly-in-vla-pipeline-weblog
No tags
f9ec9e1d836
main

docs/source/pipeline_tasks/create_docs.py

Deleted
1 -# Started from jmaster's create_docs.py.
2 -# Modified by kberry to work post-removal of the task interface.
3 -​
4 -import argparse
5 -import inspect
6 -import os
7 -import sys
8 -from collections import namedtuple
9 -from typing import Dict, Tuple
10 -​
11 -from mako.template import Template
12 -​
13 -Task = namedtuple('Task', 'name short description parameters examples')
14 -​
15 -# Task groups and their names
16 -task_groups = {"h": "Generic",
17 - "hif": "Interferometry Generic",
18 - "hifa": "Interferometry ALMA",
19 - "hifv": "Interferometry VLA",
20 - "hsd": "Single Dish",
21 - "hsdn": "Nobeyama"}
22 -​
23 -​
24 -def check_dirs(filename: str):
25 - """Pre-check/create the ancestry directories of a given file path."""
26 - filedir = os.path.dirname(filename)
27 - if not os.path.exists(filedir):
28 - os.makedirs(filedir)
29 -​
30 -​
31 -def write_landing_page(pdict, rst_file="taskdocs.rst",
32 - mako_template="pipeline_tasks.mako", outdir=None):
33 - """Creates reST file for the "landing page" for the tasks."""
34 - script_path = os.path.dirname(os.path.realpath(__file__))
35 - task_template = Template(filename=os.path.join(script_path, mako_template))
36 -​
37 - # Write the information into a rst file that can be rendered by sphinx as html/pdf/etc.
38 -​
39 - output_dir = script_path if outdir is None else outdir
40 - rst_file_full_path = os.path.join(output_dir, rst_file)
41 - check_dirs(rst_file_full_path)
42 - with open(rst_file_full_path, 'w') as fd:
43 - rst_text = task_template.render(plversion=2023, pdict=pdict, task_groups=task_groups)
44 - fd.writelines(rst_text)
45 -​
46 -​
47 -def write_task_pages(pdict, outdir=None):
48 - """Creates reST files for each task.
49 - """
50 - script_path = os.path.dirname(os.path.realpath(__file__))
51 - task_template = Template(filename=os.path.join(script_path, 'individual_task.mako'))
52 -​
53 - output_dir = script_path if outdir is None else outdir
54 - for entry in pdict:
55 - for task in pdict[entry]:
56 - rst_file = "{}/{}_task.rst".format(entry, task.name)
57 - rst_file_full_path = os.path.join(output_dir, rst_file)
58 - check_dirs(rst_file_full_path)
59 - with open(rst_file_full_path, 'w') as fd:
60 - rst_text = task_template.render(category=entry, name=task.name, description=task.description,
61 - parameters=task.parameters, examples=task.examples)
62 - fd.writelines(rst_text)
63 -​
64 -​
65 -def _parse_description(description_section: str) -> Tuple[str, str]:
66 - """ Parse the short and long descriptions from the docstring """
67 - short_description = ""
68 - long_description = ""
69 -​
70 - index = 0
71 - lines = description_section.split("\n")
72 - if len(lines) > 1:
73 - short_description = lines[1].split("----")
74 - if len(short_description) > 1:
75 - if short_description[1] != '':
76 - short_description = short_description[1]
77 - index = 2
78 - else:
79 - # hifa_wvrgcal and hifa_wvrgcal flag have longer short
80 - # descriptions that extend onto the next line
81 - short_description = lines[2].strip() + "\n" + lines[3].strip()
82 - index = 4
83 -​
84 - long_description = description_section
85 -​
86 - # Better format long description:
87 - long_split = long_description.split('\n')[index:]
88 - long_split_stripped = [line[4:] for line in long_split]
89 - long_description = "\n".join(long_split_stripped).strip("\n")
90 -​
91 - return short_description, long_description
92 -​
93 -​
94 -def _parse_parameters(parameters_section: str) -> Dict[str, str]:
95 - """
96 - Parse the parameters section of the docstring and return a
97 - dict of {'parameter': 'description'}.
98 - """
99 - parms_split = parameters_section.split("\n")
100 -​
101 - parameters_dict = {} # format is {'parameter': 'description'}
102 - current_parm_desc = None
103 - parameter_name = ""
104 - for line in parms_split:
105 - if len(line) > 4:
106 - if not line[4].isspace():
107 - if current_parm_desc is not None:
108 - parameters_dict[parameter_name] = current_parm_desc
109 - parameter_name = line.split()[0]
110 - index = line.find(parameter_name)
111 - description_line_one = line[index+len(parameter_name):].strip()
112 - current_parm_desc = description_line_one
113 - else:
114 - if current_parm_desc is not None:
115 - new_line = line.strip()
116 - # Don't add totally empty lines:
117 - if not new_line.isspace():
118 - current_parm_desc = current_parm_desc + " " + new_line + "\n"
119 -​
120 - # Add the information for the last parameter
121 - if parameter_name != "" and current_parm_desc is not None:
122 - parameters_dict[parameter_name] = current_parm_desc
123 -​
124 - return parameters_dict
125 -​
126 -​
127 -def _parse_examples(examples_section: str) -> str:
128 - """ Parse examples section from the docstring """
129 - examples = examples_section.strip("\n")
130 -​
131 - # There are 4 spaces before the text on each line begins for the
132 - # examples and there can be leading and trailing lines with only
133 - # newlines, which are stripped out.
134 - examples = "\n".join([line[4:] for line in examples.split("\n")]).strip("\n")
135 - return examples
136 -​
137 -​
138 -def docstring_parse(docstring: str) -> Tuple[str, str, str, dict]:
139 - """ Parses the docstring for each pipeline task.
140 -​
141 - This will parse the non-standard docstring format currently used
142 - for pipeline tasks and return the short description, the long
143 - description, the examples, and a dictionary of
144 - {'parameter name' : 'parameter description'}
145 -​
146 - If parsing something fails, it will continue and a warning message will
147 - be printed to stdout.
148 -​
149 - Example of the non-standard docstring-format that this will parse:
150 -​
151 - h_example_task ---- An example task short description
152 -​
153 - h_example task is an example task that serves as an example of
154 - the non-standard docstring format parsed by this script, and this
155 - is the long description.
156 -​
157 - --------- parameter descriptions ---------------------------------------------
158 -​
159 - filename A filename that could be set as input if this were a real
160 - task.
161 - example: filename='filename.txt'
162 - optional An optional parameter that can be set. This does nothing.
163 - example: optional=True
164 -​
165 - --------- examples -----------------------------------------------------------
166 -​
167 - 1. Run the example task
168 -​
169 - >>> h_example_task()
170 -​
171 - 2. Run the example task with the ``optional`` parameter set
172 -​
173 - >>> h_example_task(optional=True)
174 -​
175 - --------- issues -----------------------------------------------------------
176 -​
177 - This is an example task, but if it had any known issues, they would be here.
178 - """
179 - # Strings that delimit the different sections of the pipeline task docstrings:
180 - parameter_delimiter = "--------- parameter descriptions ---------------------------------------------"
181 - examples_delimiter = "--------- examples -----------------------------------------------------------"
182 - issues_delimiter = "--------- issues -----------------------------------------------------------"
183 -​
184 - try:
185 - description_section, rest_of_docstring = docstring.split(parameter_delimiter)
186 -​
187 - short_description, long_description = _parse_description(description_section)
188 -​
189 - parameters_section, examples_section = rest_of_docstring.split(examples_delimiter)
190 -​
191 - parameters_dict = _parse_parameters(parameters_section)
192 -​
193 - # The "issues" section is excluded from the output docs and is not
194 - # always present. If present, it will always be the last
195 - # section.
196 - if issues_delimiter in examples_section:
197 - temp_split = examples_section.split(issues_delimiter)
198 - examples_section = temp_split[0]
199 -​
200 - examples = _parse_examples(examples_section)
201 -​
202 - except Exception as e:
203 - print("Failed to parse docstring. Error: {}".format(e))
204 - print("Failing docstring: {}".format(docstring))
205 -​
206 - return short_description, long_description, examples, parameters_dict
207 -​
208 -​
209 -def create_docs(outdir=None, srcdir=None, missing_report=False, tasks_to_exclude=None):
210 - """
211 - Walks through the pipeline and creates reST documentation for each pipeline task, including an
212 - overall landing page.
213 -​
214 - Optionally generates and outputs lists of tasks with missing examples, parameters, and
215 - longer descriptions.
216 - """
217 - if srcdir is not None and os.path.exists(srcdir):
218 - sys.path.insert(0, srcdir)
219 - try:
220 - import pipeline.cli
221 - except ImportError:
222 - raise ImportError("Can not import the Pipeline package to inspect the task docs.")
223 -​
224 - # Dict which stores { 'task group' : [list of Tasks in that group]}
225 - tasks_by_group = {"h": [],
226 - "hif": [],
227 - "hifa": [],
228 - "hifv": [],
229 - "hsd": [],
230 - "hsdn": []}
231 -​
232 - if not tasks_to_exclude:
233 - # Tasks to exclude from the reference manual
234 - # hifv tasks confirmed by John Tobin via email 20230911
235 - # h tasks requested by Remy via email 20230921
236 - tasks_to_exclude = ['h_applycal',
237 - 'h_export_calstate',
238 - 'h_exportdata',
239 - 'h_import_calstate',
240 - 'h_importdata',
241 - 'h_mssplit',
242 - 'h_restoredata',
243 - 'h_show_calstate',
244 - 'hifv_targetflag',
245 - 'hifv_gaincurves',
246 - 'hifv_opcal',
247 - 'hifv_rqcal',
248 - 'hifv_swpowcal',
249 - 'hifv_tecmaps']
250 -​
251 - # Lists of cli PL tasks that are missing various pieces:
252 - missing_example = []
253 - missing_description = []
254 - missing_parameters = []
255 -​
256 - # Walk through the whole pipeline and generate documentation for cli pipeline tasks
257 - for group_name, obj in inspect.getmembers(pipeline):
258 - if group_name in task_groups.keys():
259 - for folder_name, sub_obj in inspect.getmembers(obj):
260 - if 'cli' in folder_name:
261 - for task_name, task_func in inspect.getmembers(sub_obj):
262 - if '__' not in task_name and task_name is not None and task_name[0] == 'h':
263 - docstring = task_func.__doc__
264 - short_description, long_description, examples, parameters = docstring_parse(docstring)
265 -​
266 - if missing_report:
267 - if not examples:
268 - missing_example.append(task_name)
269 - if not long_description:
270 - missing_description.append(task_name)
271 - if not parameters:
272 - missing_parameters.append(task_name)
273 -​
274 - if task_name not in tasks_to_exclude:
275 - tasks_by_group[group_name].append(
276 - Task(task_name, short_description, long_description, parameters, examples))
277 - else:
278 - print("Excluding task: {}".format(task_name))
279 -​
280 - if missing_report:
281 - print("The following tasks are missing examples:")
282 - for name in missing_example:
283 - print(name)
284 - print("\n")
285 -​
286 - print("The following tasks are missing descriptons:")
287 - for name in missing_description:
288 - print(name)
289 - print("\n")
290 -​
291 - print("The following tasks are missing parameters:")
292 - for name in missing_parameters:
293 - print(name)
294 - print("\n")
295 -​
296 - # Write out "landing page"
297 - write_landing_page(tasks_by_group, outdir=outdir)
298 -​
299 - # Write individual task pages
300 - write_task_pages(tasks_by_group, outdir=outdir)
301 -​
302 -​
303 -def cli_command():
304 - """CLI interface of create_docs.py.
305 -​
306 - try `python create_docs.py --help`
307 - """
308 -​
309 - parser = argparse.ArgumentParser(description='Generate Pipeline task .RST files')
310 - parser.add_argument('--outdir', '-o', type=str, default=None, help='Output path of the RST files/subdirectories')
311 - parser.add_argument('--srcdir', '-s', type=str, default=None, help='Path of the Pipeline source code')
312 -​
313 - args = parser.parse_args()
314 - srcdir = args.srcdir
315 -​
316 - # the primary fallback default of the pipeline source directory.
317 - env_pipeline_src = os.getenv('pipeline_src')
318 - if srcdir is None and env_pipeline_src:
319 - # use the env variable "pipeline_src" for the Pipeline source code path.
320 - srcdir = os.path.abspath(env_pipeline_src)
321 -​
322 - # the secondary fallback default of the Pipeline source directory.
323 - if srcdir is None:
324 - # use the ancestry path if "pipeline_dir" is not set.
325 - srcdir = os.path.abspath('../../pipeline')
326 -​
327 - create_docs(outdir=args.outdir, srcdir=srcdir, missing_report=True)
328 -​
329 -​
330 -if __name__ == "__main__":
331 - cli_command()

Everything looks good. We'll let you know here if there's anything you should know about.

Add shortcut