From ad8f72b3f65031c6a4f5d423c6490f4cf9f12dee Mon Sep 17 00:00:00 2001 From: Juhan Bae Date: Tue, 19 Mar 2024 13:35:05 -0400 Subject: [PATCH 1/3] Initial Commit --- .../librispeech_pytorch/models.py | 10 +++++----- submission_runner.py | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py b/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py index fe3a1e179..1476fd361 100644 --- a/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py +++ b/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py @@ -93,7 +93,7 @@ def __init__(self, out_features=self.encoder_dim, bias=True) self.pos_encode = AddPositionalEmbedding(embedding_dim=self.encoder_dim) - self.dropout = nn.Dropout(p=self.input_dropout_rate) + self.dropout = nn.Dropout(p=self.input_dropout_rate, inplace=True) def forward(self, inputs, input_paddings): output_paddings = input_paddings @@ -195,7 +195,7 @@ def __init__(self, config: ConformerConfig): in_features=config.encoder_dim, out_features=config.encoder_dim * config.feed_forward_expansion_factor, bias=True) - self.dropout1 = nn.Dropout(p=config.feed_forward_dropout_rate) + self.dropout1 = nn.Dropout(p=config.feed_forward_dropout_rate, inplace=True) self.linear2 = nn.Linear( in_features=config.encoder_dim * config.feed_forward_expansion_factor, out_features=config.encoder_dim, @@ -206,7 +206,7 @@ def __init__(self, config: ConformerConfig): else: feed_forward_residual_dropout_rate = ( config.feed_forward_residual_dropout_rate) - self.dropout2 = nn.Dropout(p=feed_forward_residual_dropout_rate) + self.dropout2 = nn.Dropout(p=feed_forward_residual_dropout_rate, inplace=True) def forward(self, inputs, padding_mask): inputs = self.ln(inputs) @@ -316,7 +316,7 @@ def __init__(self, config: ConformerConfig): attention_residual_dropout_rate = 0.1 else: attention_residual_dropout_rate = config.attention_residual_dropout_rate - self.dropout = nn.Dropout(p=attention_residual_dropout_rate) + self.dropout = nn.Dropout(p=attention_residual_dropout_rate, inplace=True) def forward(self, outputs, paddings): outputs = self.ln(outputs) @@ -407,7 +407,7 @@ def __init__(self, config): conv_residual_dropout_rate = 0.0 else: conv_residual_dropout_rate = config.conv_residual_dropout_rate - self.dropout = nn.Dropout(p=conv_residual_dropout_rate) + self.dropout = nn.Dropout(p=conv_residual_dropout_rate, inplace=True) def forward(self, inputs, input_paddings): inputs = self.ln(inputs) diff --git a/submission_runner.py b/submission_runner.py index ff290079b..7c8d7fb53 100644 --- a/submission_runner.py +++ b/submission_runner.py @@ -203,6 +203,7 @@ def train_once( log_dir: Optional[str] = None, save_checkpoints: Optional[bool] = True ) -> Tuple[spec.Timing, Dict[str, Any]]: + _reset_cuda_mem() data_rng, opt_init_rng, model_init_rng, rng = prng.split(rng, 4) # Workload setup. From de238bcbd7831b29886517e868ecacd3540babf9 Mon Sep 17 00:00:00 2001 From: Juhan Bae Date: Tue, 19 Mar 2024 13:41:05 -0400 Subject: [PATCH 2/3] Lint fix --- .../librispeech_conformer/librispeech_pytorch/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py b/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py index 1476fd361..b3f1eeaad 100644 --- a/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py +++ b/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py @@ -206,7 +206,8 @@ def __init__(self, config: ConformerConfig): else: feed_forward_residual_dropout_rate = ( config.feed_forward_residual_dropout_rate) - self.dropout2 = nn.Dropout(p=feed_forward_residual_dropout_rate, inplace=True) + self.dropout2 = nn.Dropout(p=feed_forward_residual_dropout_rate, + inplace=True) def forward(self, inputs, padding_mask): inputs = self.ln(inputs) From f208dd2dc13d98619633b3144e7300fd74060461 Mon Sep 17 00:00:00 2001 From: Juhan Bae Date: Tue, 19 Mar 2024 13:56:49 -0400 Subject: [PATCH 3/3] Lint fix --- .../librispeech_conformer/librispeech_pytorch/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py b/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py index b3f1eeaad..90a12b779 100644 --- a/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py +++ b/algorithmic_efficiency/workloads/librispeech_conformer/librispeech_pytorch/models.py @@ -206,9 +206,9 @@ def __init__(self, config: ConformerConfig): else: feed_forward_residual_dropout_rate = ( config.feed_forward_residual_dropout_rate) - self.dropout2 = nn.Dropout(p=feed_forward_residual_dropout_rate, - inplace=True) - + self.dropout2 = nn.Dropout( + p=feed_forward_residual_dropout_rate, inplace=True) + def forward(self, inputs, padding_mask): inputs = self.ln(inputs) inputs = self.linear1(inputs)