Add Trajectory and Policy Queues

config/sac_trainer_config.yaml

@@ -7,15 +7,15 @@ default:
     init_entcoef: 1.0
     learning_rate: 3.0e-4
     learning_rate_schedule: constant
-    max_steps: 5.0e4
+    max_steps: 5.0e5
     memory_size: 256
     normalize: false
     num_update: 1
     train_interval: 1
     num_layers: 2
     time_horizon: 64
     sequence_length: 64
-    summary_freq: 1000
+    summary_freq: 10000
     tau: 0.005
     use_recurrent: false
     vis_encode_type: simple
@@ -28,73 +28,73 @@ FoodCollector:
     normalize: false
     batch_size: 256
     buffer_size: 500000
-    max_steps: 1.0e5
+    max_steps: 2.0e6
     init_entcoef: 0.05
     train_interval: 1
 
 Bouncer:
     normalize: true
-    max_steps: 5.0e5
+    max_steps: 2.0e7
     num_layers: 2
     hidden_units: 64
-    summary_freq: 1000
+    summary_freq: 20000
 
 PushBlock:
-    max_steps: 5.0e4
+    max_steps: 1.5e7
     init_entcoef: 0.05
     hidden_units: 256
-    summary_freq: 2000
+    summary_freq: 60000
     time_horizon: 64
     num_layers: 2
 
 SmallWallJump:
-    max_steps: 1.0e6
+    max_steps: 3e7
     hidden_units: 256
-    summary_freq: 2000
+    summary_freq: 20000
     time_horizon: 128
     init_entcoef: 0.1
     num_layers: 2
     normalize: false
 
 BigWallJump:
-    max_steps: 1.0e6
+    max_steps: 3e7
     hidden_units: 256
-    summary_freq: 2000
+    summary_freq: 20000
     time_horizon: 128
     num_layers: 2
     init_entcoef: 0.1
     normalize: false
 
 Striker:
-    max_steps: 5.0e5
+    max_steps: 5.0e6
     learning_rate: 1e-3
     hidden_units: 256
-    summary_freq: 2000
+    summary_freq: 20000
     time_horizon: 128
     init_entcoef: 0.1
     num_layers: 2
     normalize: false
 
 Goalie:
-    max_steps: 5.0e5
+    max_steps: 5.0e6
     learning_rate: 1e-3
     hidden_units: 256
-    summary_freq: 2000
+    summary_freq: 20000
     time_horizon: 128
     init_entcoef: 0.1
     num_layers: 2
     normalize: false
 
 Pyramids:
-    summary_freq: 2000
+    summary_freq: 30000
     time_horizon: 128
     batch_size: 128
     buffer_init_steps: 10000
     buffer_size: 500000
     hidden_units: 256
     num_layers: 2
     init_entcoef: 0.01
-    max_steps: 5.0e5
+    max_steps: 1.0e7
     sequence_length: 16
     tau: 0.01
     use_recurrent: false
@@ -115,7 +115,7 @@ VisualPyramids:
     hidden_units: 256
     buffer_init_steps: 1000
     num_layers: 1
-    max_steps: 5.0e5
+    max_steps: 1.0e7
     buffer_size: 500000
     init_entcoef: 0.01
     tau: 0.01
@@ -134,21 +134,21 @@ VisualPyramids:
     normalize: true
     batch_size: 64
     buffer_size: 12000
-    summary_freq: 1000
+    summary_freq: 12000
     time_horizon: 1000
     hidden_units: 64
     init_entcoef: 0.5
 
 3DBallHard:
     normalize: true
     batch_size: 256
-    summary_freq: 1000
+    summary_freq: 12000
     time_horizon: 1000
 
 Tennis:
     buffer_size: 500000
     normalize: true
-    max_steps: 2e5
+    max_steps: 4e6
 
 CrawlerStatic:
     normalize: true
@@ -157,8 +157,8 @@ CrawlerStatic:
     train_interval: 2
     buffer_size: 500000
     buffer_init_steps: 2000
-    max_steps: 5e5
-    summary_freq: 3000
+    max_steps: 5e6
+    summary_freq: 30000
     init_entcoef: 1.0
     num_layers: 3
     hidden_units: 512
@@ -172,10 +172,10 @@ CrawlerDynamic:
     time_horizon: 1000
     batch_size: 256
     buffer_size: 500000
-    summary_freq: 3000
+    summary_freq: 30000
     train_interval: 2
     num_layers: 3
-    max_steps: 1e6
+    max_steps: 1e7
     hidden_units: 512
     reward_signals:
         extrinsic:
@@ -187,8 +187,8 @@ Walker:
     time_horizon: 1000
     batch_size: 256
     buffer_size: 500000
-    max_steps: 2e6
-    summary_freq: 3000
+    max_steps: 2e7
+    summary_freq: 30000
     num_layers: 4
     train_interval: 2
     hidden_units: 512
@@ -202,16 +202,16 @@ Reacher:
     time_horizon: 1000
     batch_size: 128
     buffer_size: 500000
-    max_steps: 2e5
-    summary_freq: 3000
+    max_steps: 2e7
+    summary_freq: 60000
 
 Hallway:
     sequence_length: 32
     num_layers: 2
     hidden_units: 128
     memory_size: 256
     init_entcoef: 0.1
-    max_steps: 5.0e5
+    max_steps: 1.0e7
     summary_freq: 1000
     time_horizon: 64
     use_recurrent: true
@@ -223,8 +223,7 @@ VisualHallway:
     memory_size: 256
     gamma: 0.99
     batch_size: 64
-    max_steps: 5.0e5
-    summary_freq: 1000
+    max_steps: 1.0e7
     time_horizon: 64
     use_recurrent: true
 
@@ -237,8 +236,8 @@ VisualPushBlock:
     gamma: 0.99
     buffer_size: 1024
     batch_size: 64
-    max_steps: 5.0e5
-    summary_freq: 1000
+    max_steps: 3.0e6
+    summary_freq: 60000
     time_horizon: 64
 
 GridWorld:
@@ -249,8 +248,8 @@ GridWorld:
     init_entcoef: 0.5
     buffer_init_steps: 1000
     buffer_size: 50000
-    max_steps: 50000
-    summary_freq: 2000
+    max_steps: 500000
+    summary_freq: 20000
     time_horizon: 5
     reward_signals:
         extrinsic: