Merge pull request #416 from prabhatnagarajan/batch_ddpg

muupan · web-flow · commit b8c72c956dfc · 2019-04-13T02:51:54.000+09:00
Enables batch DDPG agents to be trained.
diff --git a/chainerrl/agents/ddpg.py b/chainerrl/agents/ddpg.py
@@ -13,8 +13,8 @@
 from chainer import cuda
 import chainer.functions as F
 
-from chainerrl.agent import Agent
 from chainerrl.agent import AttributeSavingMixin
+from chainerrl.agent import BatchAgent
 from chainerrl.misc.batch_states import batch_states
 from chainerrl.misc.copy_param import synchronize_parameters
 from chainerrl.recurrent import Recurrent
@@ -40,7 +40,7 @@ def __init__(self, policy, q_func):
         super().__init__(policy=policy, q_function=q_func)
 
 
-class DDPG(AttributeSavingMixin, Agent):
+class DDPG(AttributeSavingMixin, BatchAgent):
     """Deep Deterministic Policy Gradients.
 
     This can be used as SVG(0) by specifying a Gaussian policy instead of a
@@ -178,7 +178,6 @@ def compute_critic_loss(self, batch):
         batch_terminal = batch['is_state_terminal']
         batch_state = batch['state']
         batch_actions = batch['action']
-        batch_next_actions = batch['next_action']
         batchsize = len(batch_rewards)
 
         with chainer.no_backprop_mode():
@@ -193,6 +192,7 @@ def compute_critic_loss(self, batch):
 
             # Target Q-function observes s_{t+1} and a_{t+1}
             if isinstance(self.target_q_function, Recurrent):
+                batch_next_actions = batch['next_action']
                 self.target_q_function.update_state(
                     batch_next_state, batch_next_actions)
 
@@ -344,6 +344,91 @@ def act(self, obs):
                           self.t, action.array[0], q.array)
         return cuda.to_cpu(action.array[0])
 
+    def batch_act(self, batch_obs):
+        """Select a batch of actions for evaluation.
+
+        Args:
+            batch_obs (Sequence of ~object): Observations.
+
+        Returns:
+            Sequence of ~object: Actions.
+        """
+
+        with chainer.using_config('train', False), chainer.no_backprop_mode():
+            batch_xs = self.batch_states(batch_obs, self.xp, self.phi)
+            batch_action = self.policy(batch_xs).sample()
+            # Q is not needed here, but log it just for information
+            q = self.q_function(batch_xs, batch_action)
+
+        # Update stats
+        self.average_q *= self.average_q_decay
+        self.average_q += (1 - self.average_q_decay) * float(
+            q.array.mean(axis=0))
+        self.logger.debug('t:%s a:%s q:%s',
+                          self.t, batch_action.array[0], q.array)
+        return [cuda.to_cpu(action.array) for action in batch_action]
+
+    def batch_act_and_train(self, batch_obs):
+        """Select a batch of actions for training.
+
+        Args:
+            batch_obs (Sequence of ~object): Observations.
+
+        Returns:
+            Sequence of ~object: Actions.
+        """
+
+        batch_greedy_action = self.batch_act(batch_obs)
+        batch_action = [
+            self.explorer.select_action(
+                self.t, lambda: batch_greedy_action[i])
+            for i in range(len(batch_greedy_action))]
+
+        self.batch_last_obs = list(batch_obs)
+        self.batch_last_action = list(batch_action)
+
+        return batch_action
+
+    def batch_observe_and_train(
+            self, batch_obs, batch_reward, batch_done, batch_reset):
+        """Observe a batch of action consequences for training.
+
+        Args:
+            batch_obs (Sequence of ~object): Observations.
+            batch_reward (Sequence of float): Rewards.
+            batch_done (Sequence of boolean): Boolean values where True
+                indicates the current state is terminal.
+            batch_reset (Sequence of boolean): Boolean values where True
+                indicates the current episode will be reset, even if the
+                current state is not terminal.
+
+        Returns:
+            None
+        """
+        for i in range(len(batch_obs)):
+            self.t += 1
+            # Update the target network
+            if self.t % self.target_update_interval == 0:
+                self.sync_target_network()
+            if self.batch_last_obs[i] is not None:
+                assert self.batch_last_action[i] is not None
+                # Add a transition to the replay buffer
+                self.replay_buffer.append(
+                    state=self.batch_last_obs[i],
+                    action=self.batch_last_action[i],
+                    reward=batch_reward[i],
+                    next_state=batch_obs[i],
+                    next_action=None,
+                    is_state_terminal=batch_done[i],
+                )
+                if batch_reset[i] or batch_done[i]:
+                    self.batch_last_obs[i] = None
+            self.replay_updater.update_if_necessary(self.t)
+
+    def batch_observe(self, batch_obs, batch_reward,
+                      batch_done, batch_reset):
+        pass
+
     def stop_episode_and_train(self, state, reward, done=False):
 
         assert self.last_state is not None
diff --git a/chainerrl/q_functions/state_action_q_functions.py b/chainerrl/q_functions/state_action_q_functions.py
@@ -146,8 +146,7 @@ def __call__(self, state, action):
         return super().__call__(h)
 
 
-class FCBNLateActionSAQFunction(chainer.Chain, StateActionQFunction,
-                                RecurrentChainMixin):
+class FCBNLateActionSAQFunction(chainer.Chain, StateActionQFunction):
     """Fully-connected + BN (s,a)-input Q-function with late action input.
 
     Actions are not included until the second hidden layer and not normalized.
@@ -202,8 +201,7 @@ def __call__(self, state, action):
         return self.mlp(h)
 
 
-class FCLateActionSAQFunction(chainer.Chain, StateActionQFunction,
-                              RecurrentChainMixin):
+class FCLateActionSAQFunction(chainer.Chain, StateActionQFunction):
     """Fully-connected (s,a)-input Q-function with late action input.
 
     Actions are not included until the second hidden layer and not normalized.
diff --git a/examples/gym/train_ddpg_batch_gym.py b/examples/gym/train_ddpg_batch_gym.py
@@ -0,0 +1,194 @@
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()  # NOQA
+import argparse
+import sys
+
+import chainer
+from chainer import optimizers
+import gym
+from gym import spaces
+import gym.wrappers
+import numpy as np
+
+import chainerrl
+from chainerrl.agents.ddpg import DDPG
+from chainerrl.agents.ddpg import DDPGModel
+from chainerrl import experiments
+from chainerrl import explorers
+from chainerrl import misc
+from chainerrl import policy
+from chainerrl import q_functions
+from chainerrl import replay_buffer
+
+
+def main():
+    import logging
+    logging.basicConfig(level=logging.DEBUG)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--outdir', type=str, default='results',
+                        help='Directory path to save output files.'
+                             ' If it does not exist, it will be created.')
+    parser.add_argument('--env', type=str, default='Humanoid-v2')
+    parser.add_argument('--seed', type=int, default=0,
+                        help='Random seed [0, 2 ** 32)')
+    parser.add_argument('--gpu', type=int, default=0)
+    parser.add_argument('--final-exploration-steps',
+                        type=int, default=10 ** 6)
+    parser.add_argument('--actor-lr', type=float, default=1e-4)
+    parser.add_argument('--critic-lr', type=float, default=1e-3)
+    parser.add_argument('--load', type=str, default='')
+    parser.add_argument('--steps', type=int, default=10 ** 7)
+    parser.add_argument('--n-hidden-channels', type=int, default=300)
+    parser.add_argument('--n-hidden-layers', type=int, default=3)
+    parser.add_argument('--replay-start-size', type=int, default=5000)
+    parser.add_argument('--n-update-times', type=int, default=1)
+    parser.add_argument('--target-update-interval',
+                        type=int, default=1)
+    parser.add_argument('--target-update-method',
+                        type=str, default='soft', choices=['hard', 'soft'])
+    parser.add_argument('--soft-update-tau', type=float, default=1e-2)
+    parser.add_argument('--update-interval', type=int, default=4)
+    parser.add_argument('--eval-n-runs', type=int, default=100)
+    parser.add_argument('--eval-interval', type=int, default=10 ** 5)
+    parser.add_argument('--gamma', type=float, default=0.995)
+    parser.add_argument('--minibatch-size', type=int, default=200)
+    parser.add_argument('--render', action='store_true')
+    parser.add_argument('--demo', action='store_true')
+    parser.add_argument('--use-bn', action='store_true', default=False)
+    parser.add_argument('--monitor', action='store_true')
+    parser.add_argument('--reward-scale-factor', type=float, default=1e-2)
+    parser.add_argument('--num-envs', type=int, default=1)
+    args = parser.parse_args()
+
+    args.outdir = experiments.prepare_output_dir(
+        args, args.outdir, argv=sys.argv)
+    print('Output files are saved in {}'.format(args.outdir))
+
+    # Set a random seed used in ChainerRL
+    misc.set_random_seed(args.seed, gpus=(args.gpu,))
+
+    def clip_action_filter(a):
+        return np.clip(a, action_space.low, action_space.high)
+
+    def reward_filter(r):
+        return r * args.reward_scale_factor
+
+    # Set different random seeds for different subprocesses.
+    # If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
+    # If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
+    process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
+    assert process_seeds.max() < 2 ** 32
+
+    def make_env(idx, test):
+        env = gym.make(args.env)
+        # Use different random seeds for train and test envs
+        process_seed = int(process_seeds[idx])
+        env_seed = 2 ** 32 - 1 - process_seed if test else process_seed
+        env.seed(env_seed)
+        # Cast observations to float32 because our model uses float32
+        env = chainerrl.wrappers.CastObservationToFloat32(env)
+        if args.monitor:
+            env = gym.wrappers.Monitor(env, args.outdir)
+        if isinstance(env.action_space, spaces.Box):
+            misc.env_modifiers.make_action_filtered(env, clip_action_filter)
+        if not test:
+            # Scale rewards (and thus returns) to a reasonable range so that
+            # training is easier
+            env = chainerrl.wrappers.ScaleReward(env, args.reward_scale_factor)
+        if args.render and not test:
+            env = chainerrl.wrappers.Render(env)
+        return env
+
+    def make_batch_env(test):
+        return chainerrl.envs.MultiprocessVectorEnv(
+            [(lambda: make_env(idx, test))
+             for idx, env in enumerate(range(args.num_envs))])
+
+    sample_env = make_env(0, test=False)
+    timestep_limit = sample_env.spec.tags.get(
+        'wrapper_config.TimeLimit.max_episode_steps')
+
+    obs_size = np.asarray(sample_env.observation_space.shape).prod()
+    action_space = sample_env.action_space
+
+    action_size = np.asarray(action_space.shape).prod()
+    if args.use_bn:
+        q_func = q_functions.FCBNLateActionSAQFunction(
+            obs_size, action_size,
+            n_hidden_channels=args.n_hidden_channels,
+            n_hidden_layers=args.n_hidden_layers,
+            normalize_input=True)
+        pi = policy.FCBNDeterministicPolicy(
+            obs_size, action_size=action_size,
+            n_hidden_channels=args.n_hidden_channels,
+            n_hidden_layers=args.n_hidden_layers,
+            min_action=action_space.low, max_action=action_space.high,
+            bound_action=True,
+            normalize_input=True)
+    else:
+        q_func = q_functions.FCSAQFunction(
+            obs_size, action_size,
+            n_hidden_channels=args.n_hidden_channels,
+            n_hidden_layers=args.n_hidden_layers)
+        pi = policy.FCDeterministicPolicy(
+            obs_size, action_size=action_size,
+            n_hidden_channels=args.n_hidden_channels,
+            n_hidden_layers=args.n_hidden_layers,
+            min_action=action_space.low, max_action=action_space.high,
+            bound_action=True)
+    model = DDPGModel(q_func=q_func, policy=pi)
+    opt_a = optimizers.Adam(alpha=args.actor_lr)
+    opt_c = optimizers.Adam(alpha=args.critic_lr)
+    opt_a.setup(model['policy'])
+    opt_c.setup(model['q_function'])
+    opt_a.add_hook(chainer.optimizer.GradientClipping(1.0), 'hook_a')
+    opt_c.add_hook(chainer.optimizer.GradientClipping(1.0), 'hook_c')
+
+    rbuf = replay_buffer.ReplayBuffer(5 * 10 ** 5)
+
+    def random_action():
+        a = action_space.sample()
+        if isinstance(a, np.ndarray):
+            a = a.astype(np.float32)
+        return a
+
+    ou_sigma = (action_space.high - action_space.low) * 0.2
+    explorer = explorers.AdditiveOU(sigma=ou_sigma)
+    agent = DDPG(model, opt_a, opt_c, rbuf, gamma=args.gamma,
+                 explorer=explorer, replay_start_size=args.replay_start_size,
+                 target_update_method=args.target_update_method,
+                 target_update_interval=args.target_update_interval,
+                 update_interval=args.update_interval,
+                 soft_update_tau=args.soft_update_tau,
+                 n_times_update=args.n_update_times,
+                 gpu=args.gpu, minibatch_size=args.minibatch_size)
+
+    if len(args.load) > 0:
+        agent.load(args.load)
+
+    if args.demo:
+        eval_stats = experiments.eval_performance(
+            env=make_batch_env(test=True),
+            agent=agent,
+            n_steps=None,
+            n_episodes=args.eval_n_runs,
+            max_episode_len=timestep_limit)
+        print('n_runs: {} mean: {} median: {} stdev {}'.format(
+            args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
+            eval_stats['stdev']))
+    else:
+        experiments.train_agent_batch_with_evaluation(
+            agent=agent, env=make_batch_env(test=False), steps=args.steps,
+            eval_env=make_batch_env(test=True), eval_n_steps=None,
+            eval_n_episodes=args.eval_n_runs, eval_interval=args.eval_interval,
+            outdir=args.outdir,
+            max_episode_len=timestep_limit)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test_examples.sh b/test_examples.sh
@@ -93,6 +93,11 @@ python examples/gym/train_ddpg_gym.py --steps 100 --replay-start-size 50 --minib
 model=$(find $outdir/gym/ddpg -name "*_finish")
 python examples/gym/train_ddpg_gym.py --demo --load $model --eval-n-runs 1 --env Pendulum-v0 --outdir $outdir/temp --gpu $gpu
 
+# gym/ddpg batch (specify non-mujoco env to test without mujoco)
+python examples/gym/train_ddpg_batch_gym.py --steps 100 --replay-start-size 50 --minibatch-size 32 --outdir $outdir/gym/ddpg_batch --env Pendulum-v0 --gpu $gpu
+model=$(find $outdir/gym/ddpg_batch -name "*_finish")
+python examples/gym/train_ddpg_batch_gym.py --demo --load $model --eval-n-runs 1 --env Pendulum-v0 --outdir $outdir/temp --gpu $gpu
+
 # gym/reinforce
 python examples/gym/train_reinforce_gym.py --steps 100 --batchsize 1 --outdir $outdir/gym/reinforce --gpu $gpu
 model=$(find $outdir/gym/reinforce -name "*_finish")
diff --git a/tests/agents_tests/test_ddpg.py b/tests/agents_tests/test_ddpg.py
@@ -9,7 +9,10 @@
 import basetest_ddpg as base
 from chainerrl.agents.ddpg import DDPG
 
+from basetest_training import _TestBatchTrainingMixin
 
+
+# Batch training with recurrent models is currently not supported
 class TestDDPGOnContinuousPOABC(base._TestDDPGOnContinuousPOABC):
 
     def make_ddpg_agent(self, env, model, actor_opt, critic_opt, explorer,
@@ -20,7 +23,8 @@ def make_ddpg_agent(self, env, model, actor_opt, critic_opt, explorer,
                     episodic_update=True, update_interval=1)
 
 
-class TestDDPGOnContinuousABC(base._TestDDPGOnContinuousABC):
+class TestDDPGOnContinuousABC(_TestBatchTrainingMixin,
+                              base._TestDDPGOnContinuousABC):
 
     def make_ddpg_agent(self, env, model, actor_opt, critic_opt, explorer,
                         rbuf, gpu):
