docs: add documentation for workspaces + general cleanup

Author: d4l3k

.github/workflows/doc-build.yaml

@@ -42,6 +42,17 @@ jobs:
         run: |
           cd docs
           make papermill
+      - name: Coverage
+        run: |
+          set -ex
+          cd docs
+          make coverage
+        if [ "$(wc -l build/*/coverage/python.txt)" -ne 2 ]
+        then
+            cat build/*/coverage/python.txt
+            echo "missing documentation coverage!"
+            exit 1
+        fi
 
   docpush:
     runs-on: ubuntu-18.04

docs/source/basics.rst

@@ -15,6 +15,7 @@ The top level modules in TorchX are:
 5. :mod:`torchx.pipelines`: adapters that convert the given app spec to a "stage" in an ML pipeline platform
 6. :mod:`torchx.runtime`: util and abstraction libraries you can use in authoring apps (not app spec)
 7. :mod:`torchx.cli`: CLI tool
+8. :mod:`torchx.workspace`: handles patching images for remote execution
 
 Below is a UML diagram
 

docs/source/components/utils.rst

@@ -10,3 +10,4 @@ Utils
 .. autofunction:: torchx.components.utils.copy
 .. autofunction:: torchx.components.utils.python
 .. autofunction:: torchx.components.utils.booth
+.. autofunction:: binary

docs/source/conf.py

@@ -66,6 +66,12 @@
     "IPython.sphinxext.ipython_console_highlighting",
 ]
 
+# coverage options
+
+coverage_ignore_modules = [
+    "torchx.components.component_test_base",
+]
+
 # katex options
 #
 #

docs/source/index.rst

@@ -16,7 +16,7 @@ most unique applications can be serviced without customizing the whole vertical
 
 
 **GETTING STARTED?** First learn the :ref:`basic concepts<basics:Basic Concepts>` and
-follow the :ref:`quickstart guide<quickstart:Quickstart>`.
+follow the :ref:`quickstart guide<quickstart:Quickstart - Custom Components>`.
 
 .. image:: torchx_index_diag.png
 
@@ -47,8 +47,37 @@ Documentation
    quickstart.md
    cli
 
+   runner.config
+
    advanced
 
+
+Works With
+---------------
+
+.. _Schedulers:
+.. toctree::
+   :maxdepth: 1
+   :caption: Schedulers
+
+   schedulers/local
+   schedulers/docker
+   schedulers/kubernetes
+   schedulers/slurm
+   schedulers/ray
+   schedulers/aws_batch
+
+.. _Pipelines:
+.. toctree::
+   :maxdepth: 1
+   :caption: Pipelines
+
+   pipelines/kfp
+
+
+Examples
+------------
+
 .. toctree::
    :maxdepth: 1
    :caption: Examples
@@ -58,6 +87,7 @@ Documentation
    examples_pipelines/index
 
 
+
 Components Library
 ---------------------
 .. _Components:
@@ -85,28 +115,6 @@ Runtime Library
    runtime/tracking
 
 
-Works With
----------------
-
-.. _Schedulers:
-.. toctree::
-   :maxdepth: 1
-   :caption: Schedulers
-
-   schedulers/local
-   schedulers/kubernetes
-   schedulers/slurm
-   schedulers/ray
-   schedulers/aws_batch
-
-.. _Pipelines:
-.. toctree::
-   :maxdepth: 1
-   :caption: Pipelines
-
-   pipelines/kfp
-
-
 Reference
 -----------
 
@@ -118,6 +126,7 @@ Reference
    specs
    runner
    schedulers
+   workspace
    pipelines
 
 .. toctree::
@@ -126,15 +135,3 @@ Reference
 
    app_best_practices
    component_best_practices
-
-
-Experimental
----------------
-.. toctree::
-   :maxdepth: 1
-   :caption: Experimental Features
-
-   experimental/runner.config
-
-
-

docs/source/quickstart.md

@@ -12,7 +12,7 @@ jupyter:
     name: python3
 ---
 
-# Quickstart
+# Quickstart - Custom Components
 
 This is a self contained guide on how to build a simple app and component spec
 and launch it via two different schedulers.

docs/source/experimental/runner.config.rst renamed to docs/source/runner.config.rst

@@ -1,4 +1,4 @@
-(beta) .torchxconfig file
+.torchxconfig
 -----------------------------
 
 .. automodule:: torchx.runner.config
@@ -10,3 +10,7 @@ Config API Functions
 .. autofunction:: apply
 .. autofunction:: load
 .. autofunction:: dump
+.. autofunction:: find_configs
+.. autofunction:: get_configs
+.. autofunction:: get_config
+.. autofunction:: load_sections

docs/source/runtime/hpo.rst

@@ -14,4 +14,7 @@ Ax (Adaptive Experimentation)
 .. currentmodule:: torchx.runtime.hpo.ax
 
 .. autoclass:: TorchXRunner
+   :members:
+
 .. autoclass:: AppMetric
+   :members:

docs/source/schedulers/aws_batch.rst

@@ -2,7 +2,17 @@ AWS Batch
 =================
 
 .. automodule:: torchx.schedulers.aws_batch_scheduler
+
 .. currentmodule:: torchx.schedulers.aws_batch_scheduler
 
 .. autoclass:: AWSBatchScheduler
    :members:
+   :show-inheritance:
+
+.. autoclass:: BatchJob
+   :members:
+
+Reference
+~~~~~~~~~~~~
+
+.. autofunction:: create_scheduler

docs/source/schedulers/docker.rst

@@ -0,0 +1,23 @@
+Docker
+=================
+
+.. automodule:: torchx.schedulers.docker_scheduler
+
+.. currentmodule:: torchx.schedulers.docker_scheduler
+
+.. autoclass:: DockerScheduler
+   :members:
+   :show-inheritance:
+
+.. autoclass:: DockerJob
+   :members:
+
+Reference
+~~~~~~~~~~~~
+
+.. autofunction:: create_scheduler
+
+.. autoclass:: DockerContainer
+   :members:
+
+.. autofunction:: has_docker