1 files changed, 3 insertions, 3 deletions

diff --git a/train_ti.py b/train_ti.py
index e6c437e..951f3dd 100644
--- a/train_ti.py
+++ b/train_ti.py
@@ -868,11 +868,11 @@ def main():
             pass
 
     @torch.no_grad()
-    def on_clip():
+    def on_clip(lr):
         embeddings = text_encoder.text_model.embeddings.temp_token_embedding
 
         pre_norm = embeddings.weight.norm(dim=-1, keepdim=True)
-        lambda_ = min(1.0, 100 * lr_scheduler.get_last_lr()[0])
+        lambda_ = min(1.0, 100 * lr)
         embeddings.weight[:] = F.normalize(embeddings.weight, dim=-1) * (pre_norm + lambda_ * (0.4 - pre_norm))
 
     loop = partial(
@@ -991,7 +991,7 @@ def main():
                         accelerator.backward(loss)
 
                         if accelerator.sync_gradients:
-                            on_clip()
+                            on_clip(lr_scheduler.get_last_lr()[0])
 
                         optimizer.step()
                         if not accelerator.optimizer_step_was_skipped: