Docs: generate all api docs automatically

Reinstate the -W option
Several overall improvements in docs
Fixed multiple links

docs/.gitignore

@@ -1,2 +1,3 @@
 /api/*
 jupyter_execute
+_toc.yml

docs/_config.yml

@@ -8,7 +8,7 @@
 
 #######################################################################################
 # Book settings
-title                       : Tutorials of Tianshou  # The title of the book. Will be placed in the left navbar.
+title                       : Tianshou Documentation  # The title of the book. Will be placed in the left navbar.
 author                      : Tianshou contributors  # The author of the book
 copyright                   : "2020, Tianshou contributors."  # Copyright year to be placed in the footer
 logo                        : _static/images/tianshou-logo.png  # A path to the book logo
@@ -88,7 +88,7 @@ launch_buttons:
 repository:
   url                       : https://github.com/carlocagnetta/tianshou.git  # The URL to your book's repository
   path_to_book              : ./docs/  # A path to your book's folder, relative to the repository root.
-  branch                    : docs/include-notebooks  # Which branch of the repository should be used when creating links
+  branch                    : master  # Which branch of the repository should be used when creating links
 
 #######################################################################################
 # Advanced and power-user settings
@@ -97,6 +97,9 @@ sphinx:
   local_extensions          :   # A list of local extensions to load by sphinx specified by "name: path" items
   recursive_update          : false # A boolean indicating whether to overwrite the Sphinx config (true) or recursively update (false)
   config                    :   # key-value pairs to directly over-ride the Sphinx configuration
+    autodoc_typehints_format: "short"
+    add_module_names: False
+    python_use_unqualified_type_names: True
     nb_mime_priority_overrides: [
       [ 'html', 'application/vnd.jupyter.widget-view+json', 10 ],
       [ 'html', 'application/javascript', 20 ],

docs/_toc.yml

@@ -1,19 +0,0 @@
-# Table of contents
-# Learn more at https://jupyterbook.org/customize/toc.html
-
-format: jb-book
-root: index
-parts:
-- caption: Get started!
-  chapters:
-    - glob: notebooks/*
-- caption: Tutorials
-  chapters:
-    - glob: tutorials/*
-- caption: API
-  chapters:
-  - glob: api/*
-- caption: Community
-  chapters:
-  - file: contributing
-  - file: contributor

docs/autogen_rst.py

@@ -6,11 +6,16 @@ from pathlib import Path
 log = logging.getLogger(os.path.basename(__file__))
 
 
+def capitalize(s: str):
+    return s[0].upper() + s[1:]
+
+
 def module_template(module_qualname: str):
     module_name = module_qualname.split(".")[-1]
     title = module_name.replace("_", r"\_")
+    title = capitalize(title)
     return f"""{title}
-{"="*len(title)}
+{"=" * len(title)}
 
 .. automodule:: {module_qualname}
    :members:
@@ -18,34 +23,16 @@ def module_template(module_qualname: str):
 """
 
 
-def package_template(package_qualname: str):
+def index_template(package_name: str, doc_references: list[str] | None = None):
-    package_name = package_qualname.split(".")[-1]
+    doc_references = doc_references or ""
-    title = package_name.replace("_", r"\_")
+    if doc_references:
-    return f"""{title}
+        doc_references = "\n" + "\n".join(f"* :doc:`{ref}`" for ref in doc_references) + "\n"
-{"="*len(title)}
 
-.. automodule:: {package_qualname}
+    dirname = package_name.split(".")[-1]
-   :members:
+    title = dirname.replace("_", r"\_")
-   :undoc-members:
+    if title == "tianshou":
-
+        title = "Tianshou API Reference"
-   {package_name}/*
+    return f"{title}\n{'=' * len(title)}" + doc_references
-"""
-
-
-def indexTemplate(package_name):
-    title = package_name
-    return f"""{title}
-{"="*len(title)}
-
-.. automodule:: {package_name}
-   :members:
-   :undoc-members:
-
-.. toctree::
-   :glob:
-
-   *
-"""
 
 
 def write_to_file(content: str, path: str):
@@ -76,7 +63,7 @@ def make_rst(src_root, rst_root, clean=False, overwrite=False, package_prefix=""
         shutil.rmtree(rst_root)
 
     base_package_name = package_prefix + os.path.basename(src_root)
-    write_to_file(indexTemplate(base_package_name), os.path.join(rst_root, "index.rst"))
+    write_to_file(index_template(base_package_name), os.path.join(rst_root, "index.rst"))
 
     for root, dirnames, filenames in os.walk(src_root):
         if os.path.basename(root).startswith("_"):
@@ -88,11 +75,33 @@ def make_rst(src_root, rst_root, clean=False, overwrite=False, package_prefix=""
         ).replace(os.path.sep, ".")
 
         for dirname in dirnames:
-            if not dirname.startswith("_"):
+            if dirname.startswith("_"):
-                package_qualname = f"{base_package_qualname}.{dirname}"
+                log.debug(f"Skipping {dirname}")
-                package_rst_path = os.path.join(rst_root, base_package_relpath, f"{dirname}.rst")
+                continue
-                log.info(f"Writing package documentation to {package_rst_path}")
+            files_in_dir = os.listdir(os.path.join(root, dirname))
-                write_to_file(package_template(package_qualname), package_rst_path)
+            module_names = [
+                f[:-3] for f in files_in_dir if f.endswith(".py") and not f.startswith("_")
+            ]
+            subdir_refs = [
+                os.path.join(f, "index")
+                for f in files_in_dir
+                if os.path.isdir(os.path.join(root, dirname, f)) and not f.startswith("_")
+            ]
+            if not module_names:
+                log.debug(f"Skipping {dirname} as it does not contain any .py files")
+                continue
+            package_qualname = f"{base_package_qualname}.{dirname}"
+            package_index_rst_path = os.path.join(
+                rst_root,
+                base_package_relpath,
+                dirname,
+                "index.rst",
+            )
+            log.info(f"Writing {package_index_rst_path}")
+            write_to_file(
+                index_template(package_qualname, doc_references=module_names + subdir_refs),
+                package_index_rst_path,
+            )
 
         for filename in filenames:
             base_name, ext = os.path.splitext(filename)
@@ -112,6 +121,6 @@ if __name__ == "__main__":
     docs_root = Path(__file__).parent
     make_rst(
         docs_root / ".." / "tianshou",
-        docs_root / "api" ,
+        docs_root / "api",
         clean=True,
     )

docs/contributor.rst

@@ -1,5 +1,5 @@
 Contributors
-===========
+============
 
 We always welcome contributions to help make Tianshou better. Below are an incomplete list of our contributors (find more on `this page <https://github.com/thu-ml/tianshou/graphs/contributors>`_).
 

docs/index.rst

@@ -52,7 +52,7 @@ Here is Tianshou's other features:
 * Support any type of environment state/action (e.g. a dict, a self-defined class, ...): :ref:`self_defined_env`
 * Support :ref:`customize_training`
 * Support n-step returns estimation :meth:`~tianshou.policy.BasePolicy.compute_nstep_return` and prioritized experience replay :class:`~tianshou.data.PrioritizedReplayBuffer` for all Q-learning based algorithms; GAE, nstep and PER are very fast thanks to numba jit function and vectorized numpy operation
-* Support :doc:`/tutorials/tictactoe`
+* Support :doc:`/tutorials/04_tictactoe`
 * Support both `TensorBoard <https://www.tensorflow.org/tensorboard>`_ and `W&B <https://wandb.ai/>`_ log tools
 * Support multi-GPU training :ref:`multi_gpu`
 * Comprehensive `unit tests <https://github.com/thu-ml/tianshou/actions>`_, including functional checking, RL pipeline checking, documentation checking, PEP8 code-style checking, and type checking
@@ -94,39 +94,6 @@ If no error occurs, you have successfully installed Tianshou.
 
 Tianshou is still under development, you can also check out the documents in stable version through `tianshou.readthedocs.io/en/stable/ <https://tianshou.readthedocs.io/en/stable/>`_.
 
-.. toctree::
-   :maxdepth: 2
-   :caption: Get started:
-   :glob:
-
-   notebooks/*
-
-
-.. toctree::
-   :maxdepth: 1
-   :caption: Tutorials
-   :glob:
-
-   tutorials/*
-
-
-.. toctree::
-   :maxdepth: 1
-   :caption: API Docs
-   :glob:
-
-
-   api/*
-
-
-.. toctree::
-   :maxdepth: 1
-   :caption: Community
-
-   contributing
-   contributor
-
-
 Indices and tables
 ------------------
 

docs/notebooks/0_intro.md

@@ -1,3 +1,4 @@
-# Tianshou Tutorials
+# Notebook Tutorials
 
-Placeholder for intro
+Here is a collection of executable tutorials for Tianshou. You can run them
+directly in colab, or download them and run them locally.

docs/tutorials/00_dqn.rst

@@ -308,7 +308,7 @@ Tianshou supports user-defined training code. Here is the code snippet:
         # train policy with a sampled batch data from buffer
         losses = policy.update(64, train_collector.buffer)
 
-For further usage, you can refer to the :doc:`/tutorials/cheatsheet`.
+For further usage, you can refer to the :doc:`/tutorials/07_cheatsheet`.
 
 .. rubric:: References
 

docs/tutorials/01_concepts.rst

@@ -339,7 +339,7 @@ Thus, we need a time-related interface for calculating the 2-step return. :meth:
 
 This code does not consider the done flag, so it may not work very well. It shows two ways to get :math:`s_{t + 2}` from the replay buffer easily in :meth:`~tianshou.policy.BasePolicy.process_fn`.
 
-For other method, you can check out :doc:`/api/tianshou.policy`. We give the usage of policy class a high-level explanation in :ref:`pseudocode`.
+For other method, you can check out :doc:`/api/policy/index`. We give the usage of policy class a high-level explanation in :ref:`pseudocode`.
 
 
 Collector
@@ -382,7 +382,7 @@ Trainer
 
 Once you have a collector and a policy, you can start writing the training method for your RL agent. Trainer, to be honest, is a simple wrapper. It helps you save energy for writing the training loop. You can also construct your own trainer: :ref:`customized_trainer`.
 
-Tianshou has three types of trainer: :func:`~tianshou.trainer.onpolicy_trainer` for on-policy algorithms such as Policy Gradient, :func:`~tianshou.trainer.offpolicy_trainer` for off-policy algorithms such as DQN, and :func:`~tianshou.trainer.offline_trainer` for offline algorithms such as BCQ. Please check out :doc:`/api/tianshou.trainer` for the usage.
+Tianshou has three types of trainer: :func:`~tianshou.trainer.onpolicy_trainer` for on-policy algorithms such as Policy Gradient, :func:`~tianshou.trainer.offpolicy_trainer` for off-policy algorithms such as DQN, and :func:`~tianshou.trainer.offline_trainer` for offline algorithms such as BCQ. Please check out :doc:`/api/trainer/index` for the usage.
 
 We also provide the corresponding iterator-based trainer classes :class:`~tianshou.trainer.OnpolicyTrainer`, :class:`~tianshou.trainer.OffpolicyTrainer`, :class:`~tianshou.trainer.OfflineTrainer` to facilitate users writing more flexible training logic:
 ::

docs/tutorials/07_cheatsheet.rst

@@ -126,7 +126,7 @@ The figure in the right gives an intuitive comparison among synchronous/asynchro
 .. note::
 
     The async simulation collector would cause some exceptions when used as
-    ``test_collector`` in :doc:`/api/tianshou.trainer` (related to
+    ``test_collector`` in :doc:`/api/trainer/index` (related to
     `Issue 700 <https://github.com/thu-ml/tianshou/issues/700>`_). Please use
     sync version for ``test_collector`` instead.
 
@@ -478,4 +478,4 @@ By constructing a new state ``state_ = (state, agent_id, mask)``, essentially we
     act = policy(state_)
     next_state_, reward = env.step(act)
 
-Following this idea, we write a tiny example of playing `Tic Tac Toe <https://en.wikipedia.org/wiki/Tic-tac-toe>`_ against a random player by using a Q-learning algorithm. The tutorial is at :doc:`/tutorials/tictactoe`.
+Following this idea, we write a tiny example of playing `Tic Tac Toe <https://en.wikipedia.org/wiki/Tic-tac-toe>`_ against a random player by using a Q-learning algorithm. The tutorial is at :doc:`/tutorials/04_tictactoe`.

docs/tutorials/index.rst

@@ -0,0 +1,2 @@
+Tutorials
+=========

pyproject.toml

@@ -65,9 +65,9 @@ optional = true
 black = { version = "^23.7.0", extras = ["jupyter"] }
 docstring-parser = "^0.15"
 jinja2 = "*"
+jsonargparse = "^4.24.1"
 jupyter = "^1.0.0"
 jupyter-book = "^0.15.1"
-jsonargparse = "^4.24.1"
 mypy = "^1.4.1"
 nbstripout = "^0.6.1"
 # networkx is used in a test
@@ -175,9 +175,10 @@ _poetry_sort = "poetry sort"
 _clean-nbs = "nbstripout docs/notebooks/*"
 format = ["_black_format", "_ruff_format", "_poetry_install_sort_plugin", "_poetry_sort", "_clean-nbs"]
 _autogen_rst = "python docs/autogen_rst.py"
-_spellcheck = "sphinx-build -b spelling docs docs/_build"
+_spellcheck = "sphinx-build -W -b spelling docs docs/_build"
-_sphinx_build = "sphinx-build -b html docs docs/_build"
+_sphinx_build = "sphinx-build -W -b html docs docs/_build"
+_jb_generate_toc = "bash -c \"jupyter-book toc from-project docs -e .rst -e .md -e .ipynb  > docs/_toc.yml \""
 _jb_config = "jupyter-book config sphinx docs/"
 doc-clean = "rm -rf docs/_build"
-doc-build = ["_autogen_rst", " _jb_config", "_sphinx_build"]
+doc-build = ["_autogen_rst", "_jb_generate_toc", " _jb_config", "_sphinx_build"]
 type-check = "mypy tianshou"

tianshou/data/buffer/base.py

@@ -18,7 +18,7 @@ class ReplayBuffer:
     stores all the data in a batch with circular-queue style.
 
     For the example usage of ReplayBuffer, please check out Section Buffer in
-    :doc:`/tutorials/concepts`.
+    :doc:`/tutorials/01_concepts`.
 
     :param size: the maximum size of replay buffer.
     :param stack_num: the frame-stack sampling argument, should be greater than or