Skip to content
Snippets Groups Projects
Commit 8c1ba3a3 authored by Andrew Maclean's avatar Andrew Maclean
Browse files

Merge branch 'Update-ClassesUsedInExamples' into 'master' 

Allow more flexibility in accessing different sites

See merge request !334
parents 1e05cbc2 6e369ba5
No related branches found
No related tags found
1 merge request!334Allow more flexibility in accessing different sites
Pipeline #395490 passed
Stage: prep
Stage: build-assets
Stage: deploy-assets
Stage: build-website
Hide whitespace changes
Inline Side-by-side
src/Admin/VTKClassesUsedInExamples.py
+ 38
− 39
View file @ 8c1ba3a3
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json
import re
......@@ -27,6 +26,8 @@ Note:
parser.add_argument('vtk_examples', help='The path to the VTK example source files. e.g. vtk-examples/src')
parser.add_argument('coverage_dest',
help='The path to the folder to store the coverage files. Usually web_dir/src/Coverage')
parser.add_argument('-w', '--web_site_url', default='https://examples.vtk.org/site/',
help='The web site URL.')
parser.add_argument('-c', '--columns',
help='Specify the number of columns for unused VTK classes output, default is 5.', nargs='?',
const=5, type=int, default=5)
......@@ -37,17 +38,31 @@ Note:
parser.add_argument('-f', '--format_json', help='Format the JSON file.', action='store_true')
args = parser.parse_args()
return (args.vtk_examples, args.coverage_dest, args.columns, args.excluded_columns,
return (args.vtk_examples, args.coverage_dest, args.web_site_url, args.columns, args.excluded_columns,
args.add_vtk_html, args.format_json)
@dataclass(frozen=True)
class Links:
"""
The URL's to the VTK documentation and to the vtk-examples site.
"""
vtk_docs: str = 'https://www.vtk.org/doc/nightly/html/'
vtk_examples: str = 'https://examples.vtk.org/site/'
def main():
example_source, coverage_dest, web_site_url, columns, excluded_columns, add_vtk_html, format_json = get_program_parameters()
source_path = Path(example_source)
coverage_path = Path(coverage_dest)
if not source_path.is_dir():
print(f'The path: {source_path} does not exist.')
return
if not coverage_path.is_dir():
print(f'The path: {coverage_path} does not exist.')
print(f'It will be created.')
vtk_docs_url: str = 'https://www.vtk.org/doc/nightly/html/'
if web_site_url[-1] != '/':
web_site_url += '/'
if vtk_docs_url[-1] != '/':
vtk_docs_url += '/'
vtk_classes = VTKClassesInExamples(source_path, coverage_dest, vtk_docs_url, web_site_url,
columns, excluded_columns, add_vtk_html, format_json)
vtk_classes.build_tables()
vtk_classes.generate_files()
@dataclass(frozen=True)
......@@ -112,11 +127,14 @@ class VTKClassesInExamples:
Determine what classes are being used or not used in the examples.
"""
def __init__(self, base_path: Path, output_path: Path, columns, excluded_columns, add_vtk_html, format_json):
def __init__(self, base_path: Path, output_path: Path, vtk_docs_url, web_site_url, columns, excluded_columns,
add_vtk_html, format_json):
"""
Note: base_path, output_path are Path objects.
:param base_path: The path to the VTK Examples sources, usually some_path/vtk-examples/src
:param output_path: Tha path to where the coverage files will be written.
:param vtk_docs_url: Tha HTML path to the Nightly VTK Documentation.
:param web_site_url: Tha HTML path to the website.
:param columns: When generating the classes not used table, the number of columns to use.
:param excluded_columns: When generating the excluded classes table, the number of columns to use.
:param add_vtk_html: True if the Doxygen documentation paths are to be added to the vtk classes in the tables.
......@@ -137,6 +155,8 @@ class VTKClassesInExamples:
self.output_path = output_path
else:
self.output_path = Path(output_path)
self.vtk_docs_url = vtk_docs_url
self.web_site_url = web_site_url
self.columns = columns
self.excluded_columns = excluded_columns
......@@ -172,7 +192,7 @@ class VTKClassesInExamples:
Parse the html file, getting a dict keyed on the vtk class name
and the value us the html file name defining the class.
"""
vtk_docs_link = f'{Links.vtk_docs}annotated.html'
vtk_docs_link = f'{self.vtk_docs_url}annotated.html'
print('Extracting the VTK Class data from the VTK Documentation.')
try:
f = urlopen(vtk_docs_link)
......@@ -188,7 +208,7 @@ class VTKClassesInExamples:
continue
f.close()
except IOError:
print(f'Unable to open the URL: {vtk_docs_link}')
print(f'Unable to open the URL: {self.vtk_docs_url}')
print(f' {len(self.vtk_classes)} VTK Classes found')
def get_example_file_paths(self):
......@@ -573,9 +593,6 @@ class VTKClassesInExamples:
VTK Example(s) by language(s).
"""
print('Cross-referencing VTK Classes and VTK Examples by Language')
links = Links()
vtk_examples_link = links.vtk_examples
vtk_docs_link = links.vtk_docs
for eg in self.example_types:
vtk_keys = list(sorted(list(self.classes_used[eg].keys()), key=lambda x: (x.lower(), x.swapcase())))
for vtk_class in vtk_keys:
......@@ -585,20 +602,18 @@ class VTKClassesInExamples:
tmp = dict()
for path, fn in paths.items():
for f in fn:
tmp[f] = f'{vtk_examples_link}{path}/{f}'
tmp[f] = f'{self.web_site_url}{path}/{f}'
self.vtk_examples_xref[vtk_class][eg] = tmp
for vtk_class in vtk_keys:
self.vtk_examples_xref[vtk_class]['VTKLink'] = {
vtk_class: f'{vtk_docs_link}{self.vtk_classes[vtk_class]}#details'}
vtk_class: f'{self.vtk_docs_url}{self.vtk_classes[vtk_class]}#details'}
def get_used_classes(self):
"""
Make a table of classes used for each set of examples.
"""
print('Making a table of used classes by Language.')
links = Links()
vtk_docs_link = links.vtk_docs
eg_fmt = '[{:s}]({:s})'
h1 = '# VTK Classes used in the Examples\n'
......@@ -643,7 +658,7 @@ class VTKClassesInExamples:
tmp = []
for c in list(sorted(excl_classes, key=lambda x: (x.lower(), x.swapcase()))):
if self.add_vtk_html:
tmp.append(f'[{c}]({vtk_docs_link}{self.vtk_classes[c]}#details)')
tmp.append(f'[{c}]({self.vtk_docs_url}{self.vtk_classes[c]}#details)')
else:
tmp.append(f'{c}')
if len(tmp) == self.excluded_columns:
......@@ -674,7 +689,8 @@ class VTKClassesInExamples:
f_list += tmp[k] + ' '
tmp.clear()
if self.add_vtk_html:
res.append(tr.format(f'[{c}]({vtk_docs_link}{self.vtk_classes[c]}#details)', f_list.strip()))
res.append(
tr.format(f'[{c}]({self.vtk_docs_url}{self.vtk_classes[c]}#details)', f_list.strip()))
else:
res.append(tr.format(f'{c}', f_list.strip()))
res.append('')
......@@ -685,8 +701,6 @@ class VTKClassesInExamples:
Make a table of classes that are not used for each set of examples.
"""
print('Making a table of unused classes by Language.')
links = Links()
vtk_docs_link = links.vtk_docs
h1 = '# VTK Classes not used in the Examples\n'
h2 = '## {:s}\n'
......@@ -723,7 +737,7 @@ class VTKClassesInExamples:
tmp = list()
for c in unused_classes:
if self.add_vtk_html:
tmp.append(f'[{c}]({vtk_docs_link}{self.vtk_classes[c]}#details)')
tmp.append(f'[{c}]({self.vtk_docs_url}{self.vtk_classes[c]}#details)')
else:
tmp.append(c)
idx += 1
......@@ -774,21 +788,6 @@ class VTKClassesInExamples:
fn.write_text('\n'.join(self.classes_not_used_table[eg]))
def main():
example_source, coverage_dest, columns, excluded_columns, add_vtk_html, format_json = get_program_parameters()
source_path = Path(example_source)
coverage_path = Path(coverage_dest)
if not source_path.is_dir():
print(f'The path: {source_path} does not exist.')
return
if not coverage_path.is_dir():
print(f'The path: {coverage_path} does not exist.')
print(f'It will be created.')
vtk_classes = VTKClassesInExamples(source_path, coverage_dest, columns, excluded_columns, add_vtk_html, format_json)
vtk_classes.build_tables()
vtk_classes.generate_files()
if __name__ == '__main__':
with ElapsedTime() as et:
main()
......
src/Python/Utilities/SelectExamples.md
+ 5
− 2
View file @ 8c1ba3a3
......@@ -2,6 +2,9 @@
This script allows you to obtain a list of examples corresponding to a particular VTK Class and a given language.
In order to do this a JSON file listing the vtk examples by VTK class is obtained from the gh-pages branch of the vtk-examples GitHub site. When this script runs, it checks for the existence of this JSON file in your temporary folder, downloading it if it doesn't exist. If it already exists, then it is updated if the file is more than ten minutes old.
In order to do this, a JSON file listing the vtk examples by VTK class is obtained from the gh-pages branch of the vtk-examples GitHub site. When this script runs, it checks for the existence of this JSON file in your temporary folder, downloading it if it doesn't exist. If it already exists, then it is updated if the file is more than ten minutes old.
When you run this script by specifying the VTK Class and Language (one of: `CSharp`, `Cxx`, `Java`, `Python`), a list of links to the respective examples containing that class is returned. By default five or fewer examples are returned. If there are more than five examples for a class then five examples are randomly selected. You can override the maximum number of examples (e.g. `-n20`) or select all examples `-a`.
When you run this script by specifying the VTK Class and Language (one of: `CSharp`, `Cxx`, `Java`, `Python`, `PythonicAPI`), a list of links to the respective examples containing that class is returned. By default five or fewer examples are returned. If there are more than five examples for a class then five examples are randomly selected. You can override the maximum number of examples (e.g. `-n20`) or select all examples `-a`.
!!! note
Options are also provided to force an overwrite of downloaded the JSON file (`-0`) or to change the URL to the JSON file (`-j`)
src/Python/Utilities/SelectExamples.py
+ 25
− 20
View file @ 8c1ba3a3
......@@ -4,7 +4,6 @@ import json
import os
import random
import tempfile
from dataclasses import dataclass
from datetime import datetime
from operator import itemgetter
from pathlib import Path
......@@ -19,6 +18,13 @@ def get_program_parameters():
The JSON file needed by this script is obtained from the gh-pages branch
of the vtk-examples GitHub site.
It is stored in your tempfile directory.
If you change the URL to the JSON file, remember that there is a ten minute
wait before you can overwrite the last downloaded file. To force the download
specify -o on the command line.
Here is the URL for an alternative site for testing:
"https://raw.githubusercontent.com/ajpmaclean/web-test/gh-pages/src/Coverage/vtk_vtk-examples_xref.json"
'''
parser = argparse.ArgumentParser(description=description, epilog=epilogue,
formatter_class=argparse.RawTextHelpFormatter)
......@@ -29,18 +35,14 @@ It is stored in your tempfile directory.
parser.add_argument('-n', '--number', type=int, default=5, help='The maximum number of examples.')
parser.add_argument('-m', '--md', action="store_true",
help='Display links in markdown inline format e.g. [label](URL).')
parser.add_argument('-j', '--json_xref_url',
default='https://raw.githubusercontent.com/Kitware/vtk-examples/gh-pages/src/Coverage/vtk_vtk-examples_xref.json',
help='The URL for the JSON cross-reference file.')
parser.add_argument('-o', '--overwrite', action="store_true",
help='Force an initial download of the JSON cross-reference file.')
args = parser.parse_args()
return args.vtk_class, args.language, args.all_values, args.md, args.number
@dataclass(frozen=True)
class Links:
"""
The URL to the JSON cross reference file.
"""
xref_url: str = \
'https://raw.githubusercontent.com/Kitware/vtk-examples/gh-pages/src/Coverage/vtk_vtk-examples_xref.json'
return args.vtk_class, args.language, args.all_values, args.md, args.number, args.json_xref_url, args.overwrite
def download_file(dl_path, dl_url, overwrite=False):
......@@ -54,7 +56,7 @@ def download_file(dl_path, dl_url, overwrite=False):
"""
file_name = dl_url.split('/')[-1]
# Create necessary sub-directories in the dl_path
# Create necessary subdirectories in the dl_path
# (if they don't exist).
Path(dl_path).mkdir(parents=True, exist_ok=True)
# Download if it doesn't exist in the directory overriding if overwrite is True.
......@@ -98,31 +100,33 @@ def get_examples(d, vtk_class, lang, all_values=False, number=5, md_fmt=False):
return len(links), links
def get_crossref_dict(ref_dir):
def get_crossref_dict(ref_dir, xref_url, overwrite=False):
"""
Download and return the json cross-reference file.
This function ensures that the dictionary is recent.
:param ref_dir: The directory where the file will be downloaded
:return: The dictionary cross-referencing vtk classes to examples
:param ref_dir: The directory where the file will be downloaded.
:param xref_url: The URL for the JSON cross-reference file.
:param overwrite: If true, do a download even if the file exists.
:return: The dictionary cross-referencing vtk classes to examples.
"""
path = download_file(ref_dir, Links.xref_url, overwrite=False)
path = download_file(ref_dir, xref_url, overwrite=overwrite)
if not path.is_file():
print(f'The path: {str(path)} does not exist.')
return None
dt = datetime.today().timestamp() - os.path.getmtime(path)
# Force a new download if the time difference is > 10 minutes.
if dt > 600:
path = download_file(ref_dir, Links.xref_url, overwrite=True)
path = download_file(ref_dir, xref_url, overwrite=True)
with open(path) as json_file:
return json.load(json_file)
def main():
vtk_class, language, all_values, md, number = get_program_parameters()
vtk_class, language, all_values, md, number, xref_url, overwrite = get_program_parameters()
language = language.lower()
available_languages = {k.lower(): k for k in ['CSharp', 'Cxx', 'Java', 'Python']}
available_languages = {k.lower(): k for k in ['CSharp', 'Cxx', 'Java', 'Python', 'PythonicAPI']}
available_languages.update({'cpp': 'Cxx', 'c++': 'Cxx', 'c#': 'CSharp'})
if language not in available_languages:
print(f'The language: {language} is not available.')
......@@ -131,8 +135,9 @@ def main():
return
else:
language = available_languages[language]
xref_dict = get_crossref_dict(tempfile.gettempdir())
xref_dict = get_crossref_dict(tempfile.gettempdir(), xref_url, overwrite)
if xref_dict is None:
print('The dictionary cross-referencing vtk classes to examples was not downloaded.')
return
total_number, examples = get_examples(xref_dict, vtk_class, language, all_values=all_values, number=number,
......
src/SyncSiteWithRepo.sh
+ 2
− 1
View file @ 8c1ba3a3
......@@ -68,6 +68,7 @@ mkdir -p ${WEB_REPO_DIR}/site
mkdir -p ${WEB_REPO_DIR}/_layouts
mkdir -p ${WEB_REPO_DIR}/custom_theme
mkdir -p ${WEB_REPO_DIR}/src/Artifacts
mkdir -p ${WEB_REPO_DIR}/src/Coverage
mkdir -p ${WEB_REPO_DIR}/src/Images
mkdir -p ${WEB_REPO_DIR}/src/stylesheets
mkdir -p ${WEB_REPO_DIR}/src/SupplementaryData
......@@ -92,7 +93,7 @@ rsync -zavh src/VTKBook/Figures/ ${WEB_REPO_DIR}/src/VTKBook/Figures
cp web_gitignore ${WEB_REPO_DIR}/.gitignore
echo "3) Create coverage files"
(cd src/Admin; python ./VTKClassesUsedInExamples.py -a .. ${WEB_REPO_DIR}/src/Coverage)
(cd src/Admin; python ./VTKClassesUsedInExamples.py -a .. ${WEB_REPO_DIR}/src/Coverage -w ${WEB_SITE_URL})
echo "4) Scrape the repo building the markdown files"
rm -rf ${WEB_REPO_DIR}/docs/*
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment