Lora: Only register params with grad to optimizer

1 files changed, 7 insertions, 3 deletions

diff --git a/train_lora.py b/train_lora.py
index 8fc2d69..cf73645 100644
--- a/train_lora.py
+++ b/train_lora.py
@@ -662,9 +662,13 @@ def main():
         sample_frequency = math.ceil(num_train_epochs * (sample_frequency / args.num_train_steps))
 
     optimizer = create_optimizer(
-        itertools.chain(
-            unet.parameters(),
-            text_encoder.parameters(),
+        (
+            param
+            for param in itertools.chain(
+                unet.parameters(),
+                text_encoder.parameters(),
+            )
+            if param.requires_grad
         ),
         lr=args.learning_rate,
     )