Refactor replay based scripts (#173)

* Fix the seed issue: see #171

* Quick fix

* log `episodic_length`

* Fix #172

* Fix #148 and #172-style problem for SAC

* Add benchmark scripts

* add sac script

* Removes gradient clipping reference

* use the latest reproduction script

* Remove past reproducibility script

* update documentation

Author: vwxyzjn

benchmark/c51.sh

@@ -0,0 +1,13 @@
+poetry install
+OMP_NUM_THREADS=1 xvfb-run -a python -m cleanrl_utils.benchmark \
+    --env-ids CartPole-v1 Acrobot-v1 MountainCar-v0 \
+    --command "poetry run python cleanrl/c51.py --cuda False --track --capture-video" \
+    --num-seeds 3 \
+    --workers 9
+
+poetry install -E atari
+OMP_NUM_THREADS=1 xvfb-run -a python -m cleanrl_utils.benchmark \
+    --env-ids PongNoFrameskip-v4 BeamRiderNoFrameskip-v4 BreakoutNoFrameskip-v4 \
+    --command "poetry run python cleanrl/c51_atari.py --track --capture-video" \
+    --num-seeds 3 \
+    --workers 1

benchmark/c51/README.md

@@ -0,0 +1,7 @@
+poetry install -E "mujoco pybullet"
+python -c "import mujoco_py"
+OMP_NUM_THREADS=1 xvfb-run -a python -m cleanrl_utils.benchmark \
+    --env-ids HalfCheetah-v2 Walker2d-v2 Hopper-v2 \
+    --command "poetry run python cleanrl/ddpg_continuous_action.py --track --capture-video" \
+    --num-seeds 3 \
+    --workers 3

benchmark/c51/atari.sh

@@ -0,0 +1,13 @@
+poetry install
+OMP_NUM_THREADS=1 xvfb-run -a python -m cleanrl_utils.benchmark \
+    --env-ids CartPole-v1 Acrobot-v1 MountainCar-v0 \
+    --command "poetry run python cleanrl/dqn.py --cuda False --track --capture-video" \
+    --num-seeds 3 \
+    --workers 9
+
+poetry install -E atari
+OMP_NUM_THREADS=1 xvfb-run -a python -m cleanrl_utils.benchmark \
+    --env-ids PongNoFrameskip-v4 BeamRiderNoFrameskip-v4 BreakoutNoFrameskip-v4 \
+    --command "poetry run python cleanrl/dqn_atari.py --track --capture-video" \
+    --num-seeds 3 \
+    --workers 1