diff --git a/docs/README.md b/docs/README.md index 2693001c5..72bd04663 100644 --- a/docs/README.md +++ b/docs/README.md @@ -29,6 +29,7 @@ There are also some Python specific libraries that need to be installed. You can * sphinx-design * sphinx-rtd-dark-mode * typing-extensions +* dirsync To install using the `requirements.txt` file use the following command: diff --git a/docs/build.py b/docs/build.py index 6b7700b92..173e97d10 100755 --- a/docs/build.py +++ b/docs/build.py @@ -1,441 +1,665 @@ #!/usr/bin/env python3 +""" Generate LVGL documentation using Doxygen and Sphinx. + +The first version of this file (Apr 2021) discovered the name of +the current branch (e.g. 'master', 'release/v8.4', etc.) to support +different versions of the documentation by establishing the base URL +(used in `conf.py` and in [Edit on GitHub] links), and then ran: + +- Doxygen (to generate LVGL API XML), then +- Sphinx + +to generate the LVGL document tree. Internally, Sphinx uses `breathe` +(a Sphinx extension) to provide a bridge between Doxygen XML output and +Sphinx documentation. It also supported a command-line option `clean` +to remove generated files before starting (eliminates orphan files, +for docs that have moved or changed). + +Since then its duties have grown to include: + +- Using environment variables to convey branch names to several more + places where they are used in the docs-generating process (instead + of re-writing writing `conf.py` and a `header.rst` each time docs + were generated). These are documented where they generated below. + +- Supporting additional command-line options. + +- Generating a temporary `./docs/lv_conf.h` for Doxygen to use + (config_builder.py). + +- Supporting multiple execution platforms (which then required tokenizing + Doxygen's INPUT path in `Doxyfile` and re-writing portions that used + `sed` to generate input or modify files). + +- Adding translation and API links (requiring generating docs in a + temporary directory so that the links could be programmatically + added to each document before Sphinx was run). + +- Generating EXAMPLES page + sub-examples where applicable to individual + documents, e.g. to widget-, style-, layout-pages, etc. + +- Building PDF via latex (when working). + + +Command-Line Arguments +---------------------- +Command-line arguments have been broken down to give the user the +ability to control each individual major variation in behavior of +this script. These were added to speed up long doc-development +tasks by shortening the turn-around time between doc modification +and seeing the final .html results in a local development environment. +Finally, this script can now be used in a way such that Sphinx will +only modify changed documents, and reduce an average ~22-minute +run time to a run time that is workable for rapidly repeating doc +generation to see Sphinx formatting results quickly. + + +Normal Usage +------------ +This is the way this script is used for normal (full) docs generation. + + $ python build.py skip_latex + + +Docs-Dev Initial Docs Generation Usage +-------------------------------------- +1. Set `LVGL_FIXED_TEMP_DIR` environment variable to path to + the temporary directory you will use over and over during + document editing, without trailing directory separator. + Initially directory should not exist. + +2. $ python build.py skip_latex preserve fixed_tmp_dir + +This takes typically ~22 minutes. + + +Docs-Dev Update-Only Generation Usage +------------------------------------- +After the above has been run through once, you can thereafter +run the following until docs-development task is complete. + + $ python build.py skip_latex docs_dev update + +Generation time depends on the number of `.rst` files that +have been updated: + ++--------------+------------+---------------------------------+ +| Docs Changed | Time | Typical Time to Browser Refresh | ++==============+============+=================================+ +| 0 | 6 seconds | n/a | ++--------------+------------+---------------------------------+ +| 1 | 19 seconds | 12 seconds | ++--------------+------------+---------------------------------+ +| 5 | 28 seconds | 21 seconds | ++--------------+------------+---------------------------------+ +| 20 | 59 seconds | 52 seconds | ++--------------+------------+---------------------------------+ + + +Sphinx Doc-Regeneration Criteria +-------------------------------- +Sphinx uses the following to determine what documents get updated: + +- source-doc modification date + - Change the modification date and `sphinx-build` will re-build it. + +- full (absolute) path to the source document, including its file name + - Change the path or filename and `sphinx-build` will re-build it. + +- whether the -E option is on the `sphinx-build` command line + - -E forces `sphinx-build` to do a full re-build. + + +Argument Descriptions +--------------------- +- skip_latex + The meaning of this argument has not changed: it simply skips + attempting to generate Latex and subsequent PDF generation. +- skip_api + Skips generating API pages (this saves about 70% of build time). + This is intended to be used only during doc development to speed up + turn-around time between doc modifications and seeing final results. +- no_fresh_env + Excludes -E command-line argument to `sphinx-build`, which, when present, + forces it to generate a whole new environment (memory of what was built + previously, forcing a full rebuild). "no_fresh_env" enables a rebuild + of only docs that got updated -- Sphinx's default behavior. +- preserve (previously "develop") + Leaves temporary directory intact for docs development purposes. +- fixed_tmp_dir + If (fixed_tmp_dir and 'LVGL_FIXED_TEMP_DIR' in `os.environ`), + then this script uses the value of that that environment variable + to populate `temp_directory` instead of the normal (randomly-named) + temporary directory. This is important when getting `sphinx-build` + to ONLY rebuild updated documents, since changing the directory + from which they are generated (normally the randomly-named temp + dir) will force Sphinx to do a full-rebuild because it remembers + the doc paths from which the build was last performed. +- skip_trans + Skips adding translation links. This allows direct copying of + `.rst` files to `temp_directory` when they are updated to save time + during re-build. Final build must not include this option so that + the translation links are added at the top of each intended page. +- no_copy + Skips copying ./docs/ directory tree to `temp_directory`. + This is only honored if: + - fixed_tmp_dir == True, and + - the doc files were previously copied to the temporary directory + and thus are already present there. +- docs_dev + This is a command-line shortcut to combining these command-line args: + - no_fresh_env + - preserve + - fixed_tmp_dir + - no_copy +- update + When no_copy is active, check modification dates on `.rst` files + and re-copy the updated `./docs/` files to the temporary directory + that have later modification dates, thus updating what Sphinx uses + as input. + Warning: this wipes out translation links and API-page links that + were added in the first pass, so should only be used for doc + development -- not for final doc generation. +""" # **************************************************************************** -# IMPORTANT: If you are getting a lexer error for an example you need to check +# IMPORTANT: If you are getting a PDF-lexer error for an example, check # for extra lines at the end of the file. Only a single empty line -# is allowed!!! Ask me how long it took me to figure this out +# is allowed!!! Ask me how long it took me to figure this out. # **************************************************************************** -import sys -import os -import subprocess -import re -import example_list as ex -import doc_builder -import shutil -import tempfile -import config_builder -import add_translation -# ------------------------------------------------------------------------- -# Process args. -# -# Normal usage: -# $ python build.py skip_latex -# -# Other optional arguments are meant for doc development to speed up -# turn-around time between doc modification and seeing the final results: -# -# - skip_api -# Skips generating API pages (this saves about 70% of build time). -# This is intended to be used only during doc development to speed up -# turn-around time between doc modifications and seeing final results. -# - no_fresh_env -# excludes -E command-line argument to `sphinx-build`, which forces -# generating a whole new environment (memory of what was built -# previously, forcing a full rebuild). "no_fresh_env" enables a -# rebuild of only docs that got updated -- Sphinx's default behavior. -# - develop -# Leaves temporary directory intact for docs development purposes. -# - fixed_tmp_dir -# If (fixed_tmp_dir and 'LVGL_FIXED_TEMP_DIR' in os.environ), -# then the temporary directory in the value of that environment -# variable will be used instead of the normal (randomly-named) -# temporary directory. This is important when getting `sphinx-build` -# to ONLY rebuild updated documents, since changing the directory -# from which they are generated (normally the randomly-named temp -# dir) will force Sphinx to do a full-rebuild because it remembers -# the doc paths from which the build was last generated. -# - docs_dev -# Forces "fresh_env" to False, and "fixed_tmp_dir" to True. This is -# merely a shortcut to having both "no_fresh_env" and "fixed_tmp_dir" -# on the command line. -# - skip_trans -# Skips adding translation links. This allows direct copying of -# of .RST files to `temp_directory` when they are updated to save -# time during re-build. Final build must not include this option -# so that the translation links are added at the top of each page. -# -# With arguments [skip_latex, develop, docs_dev], Sphinx will generate -# docs from a fixed temporary directory that can be then used later from -# the LVGL ./docs/ directory like this: -# -# $ sphinx-build -b html "fixed_temp_dir" "..\out_html" -D version="9.3" -j cpu_count -# -# to only rebuild docs that have been updated. -# ------------------------------------------------------------------------- -clean = 0 -skip_latex = False -skip_api = False -fresh_env = True -develop = False -fixed_tmp_dir = False -docs_dev = False -skip_trans = False -args = sys.argv[1:] +def run(): + # Python Library Imports + import sys + import os + import re + import subprocess + import shutil + import tempfile + import dirsync + from datetime import datetime -if len(args) >= 1: - if "clean" in args: - clean = 1 - if "skip_latex" in args: - skip_latex = True - if 'skip_api' in args: - skip_api = True - if 'no_fresh_env' in args: - fresh_env = False - if 'develop' in args: - develop = True - if 'fixed_tmp_dir' in args: - fixed_tmp_dir = True - if 'docs_dev' in args: - docs_dev = True - if 'skip_trans' in args: - skip_trans = True + # LVGL Custom Imports + import example_list as ex + import doc_builder + import config_builder + import add_translation -# Arg ramifications... -# docs_dev implies no fresh_env -if docs_dev: - fresh_env = False - fixed_tmp_dir = True + # --------------------------------------------------------------------- + # Start. + # --------------------------------------------------------------------- + t1 = datetime.now() + print('Current time: ' + str(t1)) - -# ------------------------------------------------------------------------- -# Due to the modifications that take place to the documentation files -# when the documentation builds it is better to copy the source files to a -# temporary folder and modify the copies. Not setting it up this way makes it -# a real headache when making alterations that need to be committed as the -# alterations trigger the files as changed. Also, this keeps maintenance -# effort to a minimum as adding a new language translation only needs to be -# done in 2 places (add_translation.py and ./docs/_ext/link_roles.py) rather -# than once for each .rst file. -# -# The html and PDF output locations are going to remain the same as they were. -# it's just the source documentation files that are going to be copied. -# ------------------------------------------------------------------------- -if fixed_tmp_dir and 'LVGL_FIXED_TEMP_DIR' in os.environ: - temp_directory = os.environ['LVGL_FIXED_TEMP_DIR'] -else: - temp_directory = tempfile.mkdtemp(suffix='.lvgl_docs') - -print(f'Using temp directory: [{temp_directory}]') - -langs = ['en'] - -# ------------------------------------------------------------------------- -# Set up paths. -# ------------------------------------------------------------------------- -base_path = os.path.abspath(os.path.dirname(__file__)) -project_path = os.path.abspath(os.path.join(base_path, '..')) -examples_path = os.path.join(project_path, 'examples') - -lvgl_src_path = os.path.join(project_path, 'src') -latex_output_path = os.path.join(temp_directory, 'out_latex') - -pdf_src_file = os.path.join(latex_output_path, 'LVGL.pdf') -pdf_dst_file = os.path.join(temp_directory, 'LVGL.pdf') -html_src_path = temp_directory -html_dst_path = os.path.join(project_path, 'out_html') - -# ------------------------------------------------------------------------- -# Change to script directory for consistency. -# ------------------------------------------------------------------------- -os.chdir(base_path) - -# ------------------------------------------------------------------------- -# Provide a way to run an external command and abort build on error. -# ------------------------------------------------------------------------- -def cmd(s, start_dir=None): - if start_dir is None: - start_dir = os.getcwd() - - saved_dir = os.getcwd() - os.chdir(start_dir) - print("") - print(s) - print("-------------------------------------") - result = os.system(s) - os.chdir(saved_dir) - - if result != 0: - print("Exiting build due to previous error.") - sys.exit(result) - -# ------------------------------------------------------------------------- -# Get current branch name -# ------------------------------------------------------------------------- -# 03-Oct-2024: Gabor requested this be changed to a branch name -# since that will always be current, and it will fix a large number -# of broken links on the docs website. This gets used in the -# 'Edit on GitHub' links in the upper-right corner of pages. -# Original code: -# status, br = subprocess.getstatusoutput("git branch --show-current") -# _, gitcommit = subprocess.getstatusoutput("git rev-parse HEAD") -# br = re.sub(r'\* ', '', br) -status, br = subprocess.getstatusoutput("git branch --show-current") -br = re.sub(r'\* ', '', br) - -# If in an unusual branch that is not 'master' or 'release/...' -# then default to 'master'. -if '/' in br and 'release' not in br: - br = 'master' - -gitcommit = br -urlpath = re.sub('release/', '', br) - -# These environment variables are used in other scripts. -os.environ['LVGL_URLPATH'] = urlpath -os.environ['LVGL_GITCOMMIT'] = gitcommit - - -lang = "en" -print("") -print("****************") -print("Building") -print("****************") - -# Remove all previous output files if 'clean' on command line. -if clean: - print('Removing previous output files...') - # api_path = os.path.join(dname, 'API') - # xml_path = os.path.join(dname, 'xml') - # doxy_path = os.path.join(dname, 'doxygen_html') - - # if os.path.exists(api_path): - # shutil.rmtree(api_path) - - # if os.path.exists(lang): - # shutil.rmtree(lang) - - if os.path.exists(html_dst_path): - shutil.rmtree(html_dst_path) - - # if os.path.exists(xml_path): - # shutil.rmtree(xml_path) + # --------------------------------------------------------------------- + # Process args. # - # if os.path.exists(doxy_path): - # shutil.rmtree(doxy_path) - - # os.mkdir(api_path) - # os.mkdir(lang) - -# ------------------------------------------------------------------------- -# Build local lv_conf.h from lv_conf_template.h for this build only. -# ------------------------------------------------------------------------- -config_builder.run() - -# ------------------------------------------------------------------------- -# Copy files to 'temp_directory' where they will be edited -# (translation link and API links) before being used to generate new docs. -# ------------------------------------------------------------------------- -shutil.copytree('.', temp_directory, dirs_exist_ok=True) -shutil.copytree(examples_path, os.path.join(temp_directory, 'examples'), dirs_exist_ok=True) - -# ------------------------------------------------------------------------- -# Replace tokens in Doxyfile in 'temp_directory' with data from this run. -# ------------------------------------------------------------------------- -with open(os.path.join(temp_directory, 'Doxyfile'), 'rb') as f: - data = f.read().decode('utf-8') - -data = data.replace('#*#*LV_CONF_PATH*#*#', os.path.join(base_path, 'lv_conf.h')) -data = data.replace('*#*#SRC#*#*', '"{0}"'.format(lvgl_src_path)) - -with open(os.path.join(temp_directory, 'Doxyfile'), 'wb') as f: - f.write(data.encode('utf-8')) - -# ------------------------------------------------------------------------- -# Generate examples pages. -# ------------------------------------------------------------------------- -print("Generating examples...") -ex.exec(temp_directory) - -if skip_trans: - print("Skipping translation links as requested.") -else: + # With argument `docs_dev`, Sphinx will generate docs from a fixed + # temporary directory that can be then used later using the same + # command line to get Sphinx to ONLY rebuild changed documents. + # This saves a huge amount of time during long document projects. # --------------------------------------------------------------------- - # Add translation links at top of all .rst files. + # Set defaults. + clean = False + skip_latex = False + skip_api = False + fresh_sphinx_env = True + preserve = False + fixed_tmp_dir = False + skip_trans = False + no_copy = False + docs_dev = False + update = False + args = sys.argv[1:] + + for arg in args: + # We use chained `if-elif-else` instead of `match` for those on Linux + # systems that will not have the required version 3.10 of Python yet. + if arg == "clean": + clean = True + elif arg == "skip_latex": + skip_latex = True + elif arg == 'skip_api': + skip_api = True + elif arg == 'no_fresh_env': + fresh_sphinx_env = False + elif arg == 'preserve': + preserve = True + elif arg == 'fixed_tmp_dir': + fixed_tmp_dir = True + elif arg == 'skip_trans': + skip_trans = True + elif arg == 'no_copy': + no_copy = True + elif arg == 'docs_dev': + docs_dev = True + elif arg == 'update': + update = True + else: + print(f'Argument [{arg}] not recognized.') + exit(1) + + # Arg ramifications: + # docs_dev implies no_fresh_env, preserve, fixed_tmp_dir, and no_copy. + if docs_dev: + fresh_sphinx_env = False + preserve = True + fixed_tmp_dir = True + no_copy = True + # --------------------------------------------------------------------- - print("Adding translation links...") - add_translation.exec(temp_directory) - -if skip_api: - print("Skipping API generation as requested.") -else: + # Due to the modifications that take place to the documentation files + # when the documentation builds it is better to copy the source files to a + # temporary folder and modify the copies. Not setting it up this way makes it + # a real headache when making alterations that need to be committed as the + # alterations trigger the files as changed. Also, this keeps maintenance + # effort to a minimum as adding a new language translation only needs to be + # done in 2 places (add_translation.py and ./docs/_ext/link_roles.py) rather + # than once for each .rst file. + # + # The html and PDF output locations are going to remain the same as they were. + # it's just the source documentation files that are going to be copied. # --------------------------------------------------------------------- - # Generate API pages and links thereto. + if fixed_tmp_dir and 'LVGL_FIXED_TEMP_DIR' in os.environ: + temp_directory = os.environ['LVGL_FIXED_TEMP_DIR'] + else: + temp_directory = tempfile.mkdtemp(suffix='.lvgl_docs') + + print(f'Using temp directory: [{temp_directory}]') + # --------------------------------------------------------------------- - print("Running Doxygen...") - cmd('doxygen Doxyfile', temp_directory) - print("Generating API documentation .RST files...") + # Set up paths. + # --------------------------------------------------------------------- + base_path = os.path.abspath(os.path.dirname(__file__)) + project_path = os.path.abspath(os.path.join(base_path, '..')) + examples_path = os.path.join(project_path, 'examples') + lvgl_src_path = os.path.join(project_path, 'src') + latex_output_path = os.path.join(temp_directory, 'out_latex') + pdf_src_file = os.path.join(latex_output_path, 'LVGL.pdf') + pdf_dst_file = os.path.join(temp_directory, 'LVGL.pdf') + html_src_path = temp_directory + html_dst_path = os.path.join(project_path, 'out_html') - doc_builder.EMIT_WARNINGS = False + # --------------------------------------------------------------------- + # Change to script directory for consistency. + # --------------------------------------------------------------------- + os.chdir(base_path) - # Create .RST files for API pages. - doc_builder.run( - project_path, - temp_directory, - os.path.join(temp_directory, 'intro'), - os.path.join(temp_directory, 'intro', 'add-lvgl-to-your-project'), - os.path.join(temp_directory, 'details'), - os.path.join(temp_directory, 'details', 'base-widget'), - os.path.join(temp_directory, 'details', 'base-widget', 'layouts'), - os.path.join(temp_directory, 'details', 'base-widget', 'styles'), - os.path.join(temp_directory, 'details', 'debugging'), - os.path.join(temp_directory, 'details', 'integration'), - os.path.join(temp_directory, 'details', 'integration', 'bindings'), - os.path.join(temp_directory, 'details', 'integration', 'building'), - os.path.join(temp_directory, 'details', 'integration', 'chip'), - os.path.join(temp_directory, 'details', 'integration', 'driver'), - os.path.join(temp_directory, 'details', 'integration', 'driver', 'display'), - os.path.join(temp_directory, 'details', 'integration', 'driver', 'touchpad'), - os.path.join(temp_directory, 'details', 'integration', 'framework'), - os.path.join(temp_directory, 'details', 'integration', 'ide'), - os.path.join(temp_directory, 'details', 'integration', 'os'), - os.path.join(temp_directory, 'details', 'integration', 'os', 'yocto'), - os.path.join(temp_directory, 'details', 'integration', 'renderers'), - os.path.join(temp_directory, 'details', 'libs'), - os.path.join(temp_directory, 'details', 'main-components'), - os.path.join(temp_directory, 'details', 'other-components'), - os.path.join(temp_directory, 'details', 'widgets') - ) + # --------------------------------------------------------------------- + # Provide a way to run an external command and abort build on error. + # --------------------------------------------------------------------- + def cmd(s, start_dir=None): + if start_dir is None: + start_dir = os.getcwd() + + saved_dir = os.getcwd() + os.chdir(start_dir) + print("") + print(s) + print("-------------------------------------") + result = os.system(s) + os.chdir(saved_dir) + + if result != 0: + print("Exiting build due to previous error.") + sys.exit(result) + + # --------------------------------------------------------------------- + # Populate LVGL_URLPATH and LVGL_GITCOMMIT environment variables: + # - LVGL_URLPATH <= 'master' or '8.4' '9.2' etc. + # - LVGL_GITCOMMIT <= same (see 03-Oct-2024 note below). + # + # These supply input later in the doc-generation process as follows: + # + # LVGL_URLPATH is used by: + # - `conf.py` to build `html_baseurl` for Sphinx for + # - generated index + # - generated search window + # - establishing canonical page for search engines + # - `link_roles.py` to generate translation links + # - `doc_builder.py` to generate links to API pages + # + # LVGL_GITCOMMIT is used by: + # - `conf.py` => html_context['github_version'] for + # Sphinx Read-the-Docs theme to add to [Edit on GitHub] links + # - `conf.py` => repo_commit_hash for generated EXAMPLES pages for: + # - [View on GitHub] buttons (view C code examples) + # - [View on GitHub] buttons (view Python code examples) + # --------------------------------------------------------------------- + # 03-Oct-2024: Gabor requested LVGL_GITCOMMIT be changed to a branch + # name since that will always be current, and it will fix a large + # number of broken links on the docs website, since commits that + # generated docs can sometimes go away. This gets used in: + # - [Edit on GitHub] links in doc pages (via Sphinx theme), and + # - [View on GitHub] links in example pages (via `example_list.py` + # and `lv_example.py`). + # Original code: + # status, br = subprocess.getstatusoutput("git branch --show-current") + # _, gitcommit = subprocess.getstatusoutput("git rev-parse HEAD") + # br = re.sub(r'\* ', '', br) + # 're' was previously used to remove leading '* ' from current branch + # string when we were parsing output from bare `git branch` output. + # This is no longer needed with `--show-current` option now used. + # --------------------------------------------------------------------- + status, branch = subprocess.getstatusoutput("git branch --show-current") + + # If above failed (i.e. `branch` not valid), default to 'master'. + if status != 0: + branch = 'master' + elif branch == 'master': + # Expected in most cases. Nothing to change. + pass + else: + # `branch` is valid. Capture release version if in a 'release/' branch. + if branch.startswith('release/'): + branch = branch[8:] + else: + # Default to 'master'. + branch = 'master' + + os.environ['LVGL_URLPATH'] = branch + os.environ['LVGL_GITCOMMIT'] = branch + + # --------------------------------------------------------------------- + # Start doc-build process. + # --------------------------------------------------------------------- + print("") + print("****************") + print("Building") + print("****************") + + # Remove all previous output files if 'clean' on command line. + if clean: + print('Removing previous output files...') + # The below commented-out code below is being preserved + # for docs-generation development purposes. + + # api_path = os.path.join(temp_directory, 'API') + # xml_path = os.path.join(temp_directory, 'xml') + # doxy_path = os.path.join(temp_directory, 'doxygen_html') + + # if os.path.exists(api_path): + # shutil.rmtree(api_path) + + # lang = 'en' + # if os.path.exists(lang): + # shutil.rmtree(lang) + + if os.path.exists(html_dst_path): + shutil.rmtree(html_dst_path) + + # if os.path.exists(xml_path): + # shutil.rmtree(xml_path) + # + # if os.path.exists(doxy_path): + # shutil.rmtree(doxy_path) + + # os.mkdir(api_path) + # os.mkdir(lang) + + # --------------------------------------------------------------------- + # Build local lv_conf.h from lv_conf_template.h for this build only. + # --------------------------------------------------------------------- + config_builder.run() + + # --------------------------------------------------------------------- + # Provide answer to question: Can we have reasonable confidence that + # the contents of `temp_directory` already exists? + # --------------------------------------------------------------------- + def temp_dir_contents_exists(): + result = False + c1 = os.path.exists(temp_directory) + + if c1: + temp_path = os.path.join(temp_directory, 'CHANGELOG.rst') + c2 = os.path.exists(temp_path) + temp_path = os.path.join(temp_directory, 'CODING_STYLE.rst') + c3 = os.path.exists(temp_path) + temp_path = os.path.join(temp_directory, 'CONTRIBUTING.rst') + c4 = os.path.exists(temp_path) + temp_path = os.path.join(temp_directory, '_ext') + c5 = os.path.exists(temp_path) + temp_path = os.path.join(temp_directory, '_static') + c6 = os.path.exists(temp_path) + temp_path = os.path.join(temp_directory, 'details') + c7 = os.path.exists(temp_path) + temp_path = os.path.join(temp_directory, 'intro') + c8 = os.path.exists(temp_path) + temp_path = os.path.join(temp_directory, 'examples') + c9 = os.path.exists(temp_path) + result = c2 and c3 and c4 and c5 and c6 and c7 and c8 and c9 + + return result + + # --------------------------------------------------------------------- + # Copy files to 'temp_directory' where they will be edited (translation + # link and API links) before being used to generate new docs. + # --------------------------------------------------------------------- + doc_files_copied = False + if no_copy and fixed_tmp_dir and temp_dir_contents_exists(): + if update: + exclude_list = ['lv_conf.h'] + options = { + 'verbose': True, + 'create': True, + 'exclude': exclude_list + } + dirsync.sync('.', temp_directory, 'update', **options) + else: + print("Skipping copying ./docs/ directory as requested.") + else: + shutil.copytree('.', temp_directory, dirs_exist_ok=True) + shutil.copytree(examples_path, os.path.join(temp_directory, 'examples'), dirs_exist_ok=True) + doc_files_copied = True + + # --------------------------------------------------------------------- + # Replace tokens in Doxyfile in 'temp_directory' with data from this run. + # --------------------------------------------------------------------- + if doc_files_copied: + with open(os.path.join(temp_directory, 'Doxyfile'), 'rb') as f: + data = f.read().decode('utf-8') + + data = data.replace('#*#*LV_CONF_PATH*#*#', os.path.join(base_path, 'lv_conf.h')) + data = data.replace('*#*#SRC#*#*', '"{0}"'.format(lvgl_src_path)) + + with open(os.path.join(temp_directory, 'Doxyfile'), 'wb') as f: + f.write(data.encode('utf-8')) + + # ----------------------------------------------------------------- + # Generate examples pages. Include sub-pages pages that get included + # in individual documents where applicable. + # ----------------------------------------------------------------- + print("Generating examples...") + ex.exec(temp_directory) + + # ----------------------------------------------------------------- + # Add translation links. + # ----------------------------------------------------------------- + if skip_trans: + print("Skipping translation links as requested.") + else: + print("Adding translation links...") + add_translation.exec(temp_directory) + + # --------------------------------------------------------------------- + # Generate API pages and links thereto. + # --------------------------------------------------------------------- + if skip_api: + print("Skipping API generation as requested.") + else: + print("Running Doxygen...") + cmd('doxygen Doxyfile', temp_directory) + + doc_builder.EMIT_WARNINGS = False + + # Create .RST files for API pages. + doc_builder.run( + project_path, + temp_directory, + os.path.join(temp_directory, 'intro'), + os.path.join(temp_directory, 'intro', 'add-lvgl-to-your-project'), + os.path.join(temp_directory, 'details'), + os.path.join(temp_directory, 'details', 'base-widget'), + os.path.join(temp_directory, 'details', 'base-widget', 'layouts'), + os.path.join(temp_directory, 'details', 'base-widget', 'styles'), + os.path.join(temp_directory, 'details', 'debugging'), + os.path.join(temp_directory, 'details', 'integration'), + os.path.join(temp_directory, 'details', 'integration', 'bindings'), + os.path.join(temp_directory, 'details', 'integration', 'building'), + os.path.join(temp_directory, 'details', 'integration', 'chip'), + os.path.join(temp_directory, 'details', 'integration', 'driver'), + os.path.join(temp_directory, 'details', 'integration', 'driver', 'display'), + os.path.join(temp_directory, 'details', 'integration', 'driver', 'touchpad'), + os.path.join(temp_directory, 'details', 'integration', 'framework'), + os.path.join(temp_directory, 'details', 'integration', 'ide'), + os.path.join(temp_directory, 'details', 'integration', 'os'), + os.path.join(temp_directory, 'details', 'integration', 'os', 'yocto'), + os.path.join(temp_directory, 'details', 'integration', 'renderers'), + os.path.join(temp_directory, 'details', 'libs'), + os.path.join(temp_directory, 'details', 'main-components'), + os.path.join(temp_directory, 'details', 'other-components'), + os.path.join(temp_directory, 'details', 'widgets') + ) + + print('Reading Doxygen output...') + + # --------------------------------------------------------------------- + # BUILD PDF + # --------------------------------------------------------------------- + if skip_latex: + print("Skipping latex build as requested.") + else: + # Remove PDF link so PDF does not have a link to itself. + index_path = os.path.join(temp_directory, 'index.rst') + + with open(index_path, 'rb') as f: + index_data = f.read().decode('utf-8') + + # Support both Windows and Linux platforms with `os.linesep`. + pdf_link_ref_str = 'PDF version: :download:`LVGL.pdf `' + os.linesep + if pdf_link_ref_str in index_data: + index_data = index_data.replace(pdf_link_ref_str, '') + + with open(index_path, 'wb') as f: + f.write(index_data.encode('utf-8')) + + # Silly workaround to include the more or less correct + # PDF download link in the PDF + # cmd("cp -f " + lang +"/latex/LVGL.pdf LVGL.pdf | true") + src = temp_directory + dst = latex_output_path + cpu = os.cpu_count() + cmd_line = f'sphinx-build -b latex "{src}" "{dst}" -j {cpu}' + cmd(cmd_line) + + # Generate PDF. + cmd_line = 'latexmk -pdf "LVGL.tex"' + cmd(cmd_line, latex_output_path) + + # Copy the result PDF to the main directory to make + # it available for the HTML build. + shutil.copyfile(pdf_src_file, pdf_dst_file) + + # Add PDF link back in so HTML build will have it. + index_data = pdf_link_ref_str + index_data + + with open(index_path, 'wb') as f: + f.write(index_data.encode('utf-8')) + + # --------------------------------------------------------------------- + # BUILD HTML + # --------------------------------------------------------------------- + # This version of get_version() works correctly under both Linux and Windows. + # Updated to be resilient to changes in `lv_version.h` compliant with C macro syntax. + def get_version(): + path = os.path.join(project_path, 'lv_version.h') + major = '' + minor = '' + + with open(path, 'r') as file: + major_re = re.compile(r'define\s+LVGL_VERSION_MAJOR\s+(\d+)') + minor_re = re.compile(r'define\s+LVGL_VERSION_MINOR\s+(\d+)') + + for line in file.readlines(): + # Skip if line not long enough to match. + if len(line) < 28: + continue + + match = major_re.search(line) + if match is not None: + major = match[1] + else: + match = minor_re.search(line) + if match is not None: + minor = match[1] + # Exit early if we have both values. + if len(major) > 0 and len(minor) > 0: + break + + return f'{major}.{minor}' + + # Note: While it can be done (e.g. if one needs to set a stop point + # in Sphinx code for development purposes), it is NOT a good idea to + # run Sphinx from script as + # from sphinx.cmd.build import main as sphinx_build + # sphinx_args = [...] + # sphinx_build(sphinx_args) + # because it takes ~10X longer to run than `sphinx_build` executable. + # Literally > 3 hours. + + # '-E' option forces Sphinx to rebuild its environment so all docs are + # fully regenerated, even if not changed. + # Note: Sphinx runs in ./docs/, but uses `temp_directory` for input. + if fresh_sphinx_env: + print("Regenerating all files...") + env_opt = '-E' + else: + print("Regenerating only updated files...") + env_opt = '' + + ver = get_version() + src = html_src_path + dst = html_dst_path + cpu = os.cpu_count() + cmd_line = f'sphinx-build -b html "{src}" "{dst}" -D version="{ver}" {env_opt} -j {cpu}' + t2 = datetime.now() + print('Current time: ' + str(t2)) + cmd(cmd_line) + t3 = datetime.now() + print('Current time: ' + str(t3)) + print('Sphinx run time: ' + str(t3 - t2)) + + # --------------------------------------------------------------------- + # Cleanup. + # --------------------------------------------------------------------- + if preserve: + print('Temp directory: ', temp_directory) + else: + print('Removing temporary files...', temp_directory) + if os.path.exists(temp_directory): + shutil.rmtree(temp_directory) + + # --------------------------------------------------------------------- + # Remove temporary `lv_conf.h` created for this build. + # --------------------------------------------------------------------- + config_builder.cleanup() + + # --------------------------------------------------------------------- + # Indicate results. + # --------------------------------------------------------------------- + t4 = datetime.now() + print('Total run time: ' + str(t4 - t1)) + print('Output path: ', html_dst_path) + print() + print('Note: warnings about `/details/index.rst` and `/intro/index.rst`') + print(' "not being in any toctree" are expected and intentional.') + print() + print('Finished.') - print('Reading Doxygen output...') # ------------------------------------------------------------------------- -# We make sure to remove the link to the PDF before the PDF get generated -# doesn't make any sense to have a link to the PDF in the PDF. The link gets -# added if there is a PDF build so the HTML build will have the link. +# Make module importable as well as run-able. # ------------------------------------------------------------------------- -index_path = os.path.join(temp_directory, 'index.rst') - -with open(index_path, 'rb') as f: - index_data = f.read().decode('utf-8') - -if 'PDF version: :download:`LVGL.pdf `' in index_data: - index_data = index_data.replace( - 'PDF version: :download:`LVGL.pdf `\n', - '' - ) - with open(index_path, 'wb') as f: - f.write(index_data.encode('utf-8')) - -# ------------------------------------------------------------------------- -# BUILD PDF -# ------------------------------------------------------------------------- -if skip_latex: - print("Skipping latex build as requested.") -else: - # Silly workaround to include the more or less correct - # PDF download link in the PDF - # cmd("cp -f " + lang +"/latex/LVGL.pdf LVGL.pdf | true") - cmd('sphinx-build -b latex "{src}" "{dst}" -j {cpu}'.format( - src=temp_directory, - dst=latex_output_path, - cpu=os.cpu_count() - )) - - # Generate PDF - cmd('cd "{out_latex}" && latexmk -pdf "LVGL.tex"'.format( - out_latex=latex_output_path - )) - - # Copy the result PDF to the main directory to make - # it available for the HTML build - - shutil.copyfile(pdf_src_file, pdf_dst_file) - # cmd("cd out_latex && cp -f LVGL.pdf ../LVGL.pdf") - - # add the PDF link so the HTML build will have it. - index_data = 'PDF version: :download:`LVGL.pdf `\n' + index_data - - with open(index_path, 'wb') as f: - f.write(index_data.encode('utf-8')) - -# ------------------------------------------------------------------------- -# BUILD HTML -# ------------------------------------------------------------------------- -# This version of get_version() also works correctly under Windows. -def get_version(): - path = os.path.join(project_path, 'lv_version.h') - with open(path, 'rb') as fle: - d = fle.read().decode('utf-8') - - d = d.split('#define LVGL_VERSION_MAJOR', 1)[-1] - major, d = d.split('\n', 1) - d = d.split('#define LVGL_VERSION_MINOR', 1)[-1] - minor, d = d.split('\n', 1) - - # d = d.split('#define LVGL_VERSION_PATCH', 1)[-1] - # patch, d = d.split('\n', 1) - - return f'{major.strip()}.{minor.strip()}' - -# ------------------------------------------------------------------------- -# Run Sphinx after determining whether to use -E (fresh environment) -# command-line argument. -# ------------------------------------------------------------------------- -if fresh_env: - # Uses -E option (same as --fresh-env). Forces sphinx-build to rebuild sphinx - # environment so that all docs are fully regenerated, even if they have not changed. - print("Regenerating all files...") - cmd('sphinx-build -b html "{src}" "{dst}" -D version="{version}" -E -j {cpu}'.format( - src=html_src_path, - dst=html_dst_path, - version=get_version(), - cpu=os.cpu_count() - )) -else: - # Does not use -E option (same as --fresh-env). - print("Regenerating only updated files...") - cmd('sphinx-build -b html "{src}" "{dst}" -D version="{version}" -j {cpu}'.format( - src=html_src_path, - dst=html_dst_path, - version=get_version(), - cpu=os.cpu_count() - )) - -# ------------------------------------------------------------------------- -# If 'develop' was specified on command line, announce location of temp dir. -# Otherwise, remove temporary files created for the doc build. -# ------------------------------------------------------------------------- -if develop: - print('Temp directory: ', temp_directory) -else: - print('Removing temporary files...', temp_directory) - # Recursively remove generated files in `temp_directory`. - def iter_temp(p): - folders = [] - remove_folder = True - for temp_file in os.listdir(p): - temp_file = os.path.join(p, temp_file) - if os.path.isdir(temp_file): - folders.append(temp_file) - else: - try: - os.remove(temp_file) - except OSError: - remove_folder = False - - for folder in folders: - if not iter_temp(folder): - remove_folder = False - - if remove_folder: - try: - os.rmdir(p) - except OSError: - remove_folder = False - - return remove_folder - - iter_temp(temp_directory) - -# ------------------------------------------------------------------------- -# Remove temporary `lv_conf.h` created for this build. -# ------------------------------------------------------------------------- -config_builder.cleanup() - -# ------------------------------------------------------------------------- -# Indicate results. -# ------------------------------------------------------------------------- -print('Output path: ', html_dst_path) -print('Finished.') +if __name__ == '__main__': + run() diff --git a/docs/doc_builder.py b/docs/doc_builder.py index cfb5c1e04..644d21450 100644 --- a/docs/doc_builder.py +++ b/docs/doc_builder.py @@ -1345,32 +1345,85 @@ class XMLSearch(object): with open(os.path.join(temp_directory, 'Doxyfile'), 'wb') as f: f.write(data.encode('utf-8')) - status, br = subprocess.getstatusoutput("git branch") + # ----------------------------------------------------------------- + # Populate LVGL_URLPATH and LVGL_GITCOMMIT environment variables: + # - LVGL_URLPATH <= 'master' or '8.4' '9.2' etc. + # - LVGL_GITCOMMIT <= commit hash of HEAD. + # The previous version of this was populating LVGL_URLPATH with + # the multi-line list of all existing branches in the repository, + # which was not what was intended. + # ----------------------------------------------------------------- + status, branch = subprocess.getstatusoutput("git branch --show-current") _, gitcommit = subprocess.getstatusoutput("git rev-parse HEAD") - br = re.sub(r'\* ', '', br) - urlpath = re.sub('release/', '', br) + # If above failed (i.e. `branch` not valid), default to 'master'. + if status != 0: + branch = 'master' + elif branch == 'master': + # Expected in most cases. Nothing to change. + pass + else: + # `branch` is valid. Capture release version if in a 'release/' branch. + if branch.startswith('release/'): + branch = branch[8:] + else: + # Default to 'master'. + branch = 'master' - os.environ['LVGL_URLPATH'] = urlpath + os.environ['LVGL_URLPATH'] = branch os.environ['LVGL_GITCOMMIT'] = gitcommit - p = subprocess.Popen( - f'cd "{temp_directory}" && doxygen Doxyfile', - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True - ) + # --------------------------------------------------------------------- + # Provide a way to run an external command and abort build on error. + # + # This is necessary because when tempdir created by tempfile.mkdtemp()` + # is on a different drive, the "cd tmpdir && doxygen Doxyfile" syntax + # fails because of the different semantics of the `cd` command on + # Windows: it doesn't change the default DRIVE if `cd` is executed + # from a different drive. The result, when this is the case, is that + # Doxygen runs in the current working directory instead of in the + # temporary directory as was intended. + # --------------------------------------------------------------------- + def cmd(cmd_str, start_dir=None): + if start_dir is None: + start_dir = os.getcwd() - out, err = p.communicate() - if p.returncode: - if out: - sys.stdout.write(out) - sys.stdout.flush() - if err: - sys.stderr.write(err) - sys.stdout.flush() + saved_dir = os.getcwd() + os.chdir(start_dir) - sys.exit(p.returncode) + # This method of running Doxygen is used because if it + # succeeds, we do not want anything going to STDOUT. + # Running it via `os.system()` would send its output + # to STDOUT. + p = subprocess.Popen( + cmd_str, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True + ) + + out, err = p.communicate() + if p.returncode: + if out: + # Note the `.decode("utf-8")` is required here + # because `sys.stdout.write()` requires a string, + # and `out` by itself is a byte array -- it causes + # it to generate an exception and abort the script. + sys.stdout.write(out.decode("utf-8")) + sys.stdout.flush() + if err: + sys.stderr.write(err.decode("utf-8")) + sys.stdout.flush() + + sys.exit(p.returncode) + + # If execution arrived here, Doxygen exited with code 0. + os.chdir(saved_dir) + + # ----------------------------------------------------------------- + # Run Doxygen in temporary directory. + # ----------------------------------------------------------------- + cmd('doxygen Doxyfile', temp_directory) xml_path = os.path.join(temp_directory, 'xml') @@ -1418,6 +1471,11 @@ class XMLSearch(object): return defines.get(m_name, None) +def announce(*args): + args = ' '.join(repr(arg) for arg in args) + print(f'{os.path.basename(__file__)}: ', args) + + def run(project_path, temp_directory, *doc_paths): """ This function does 2 things: @@ -1443,6 +1501,8 @@ def run(project_path, temp_directory, *doc_paths): api_path = os.path.join(base_path, 'API') lvgl_src_path = os.path.join(project_path, 'src') + announce("Generating API documentation .RST files...") + if not os.path.exists(api_path): os.makedirs(api_path) @@ -1463,6 +1523,8 @@ def run(project_path, temp_directory, *doc_paths): # - unions, # - typedefs, # - functions. + announce("Building source-code symbol tables...") + for compound in index: compound.attrib['name'] = compound[0].text.strip() if compound.attrib['kind'] in ('example', 'page', 'dir'): @@ -1510,6 +1572,8 @@ def run(project_path, temp_directory, *doc_paths): ) # For each directory entry in `doc_paths` array... + announce("Adding API-page hyperlinks to source docs...") + for folder in doc_paths: # Fetch a list of '.rst' files excluding 'index.rst'. rst_files = list( diff --git a/docs/requirements.txt b/docs/requirements.txt index 138de0cec..520b5c9fa 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -15,3 +15,4 @@ sphinx-design sphinx-rtd-dark-mode typing-extensions sphinx-reredirects +dirsync \ No newline at end of file