Fix sample gen: models sometimes weren't in eval mode

1 files changed, 13 insertions, 14 deletions

diff --git a/training/strategy/dreambooth.py b/training/strategy/dreambooth.py
index 42624cd..7cdfc7f 100644
--- a/training/strategy/dreambooth.py
+++ b/training/strategy/dreambooth.py
@@ -113,7 +113,7 @@ def dreambooth_strategy_callbacks(
         accelerator.clip_grad_norm_(itertools.chain(*params_to_clip), max_grad_norm)
 
     @torch.no_grad()
-    def on_after_optimize(_, lr: float):
+    def on_after_optimize(_, lrs: dict[str, float]):
         if ema_unet is not None:
             ema_unet.step(unet.parameters())
 
@@ -149,25 +149,24 @@ def dreambooth_strategy_callbacks(
         if torch.cuda.is_available():
             torch.cuda.empty_cache()
 
-    @torch.no_grad()
+    @on_eval()
     def on_sample(step):
-        with ema_context():
-            unet_ = accelerator.unwrap_model(unet, keep_fp32_wrapper=True)
-            text_encoder_ = accelerator.unwrap_model(text_encoder, keep_fp32_wrapper=True)
+        unet_ = accelerator.unwrap_model(unet, keep_fp32_wrapper=True)
+        text_encoder_ = accelerator.unwrap_model(text_encoder, keep_fp32_wrapper=True)
 
-            orig_unet_dtype = unet_.dtype
-            orig_text_encoder_dtype = text_encoder_.dtype
+        orig_unet_dtype = unet_.dtype
+        orig_text_encoder_dtype = text_encoder_.dtype
 
-            unet_.to(dtype=weight_dtype)
-            text_encoder_.to(dtype=weight_dtype)
+        unet_.to(dtype=weight_dtype)
+        text_encoder_.to(dtype=weight_dtype)
 
-            save_samples_(step=step, unet=unet_, text_encoder=text_encoder_)
+        save_samples_(step=step, unet=unet_, text_encoder=text_encoder_)
 
-            unet_.to(dtype=orig_unet_dtype)
-            text_encoder_.to(dtype=orig_text_encoder_dtype)
+        unet_.to(dtype=orig_unet_dtype)
+        text_encoder_.to(dtype=orig_text_encoder_dtype)
 
-            del unet_
-            del text_encoder_
+        del unet_
+        del text_encoder_
 
         if torch.cuda.is_available():
             torch.cuda.empty_cache()