1 files changed, 16 insertions, 6 deletions

diff --git a/train_ti.py b/train_ti.py
index df8d443..35be74c 100644
--- a/train_ti.py
+++ b/train_ti.py
@@ -169,6 +169,11 @@ def parse_args():
         help="Tag dropout probability.",
     )
     parser.add_argument(
+        "--tag_shuffle",
+        type="store_true",
+        help="Shuffle tags.",
+    )
+    parser.add_argument(
         "--vector_dropout",
         type=int,
         default=0,
@@ -395,7 +400,7 @@ def parse_args():
     parser.add_argument(
         "--sample_steps",
         type=int,
-        default=15,
+        default=20,
         help="Number of steps for sample generation. Higher values will result in more detailed samples, but longer runtimes.",
     )
     parser.add_argument(
@@ -745,6 +750,7 @@ def main():
         bucket_step_size=args.bucket_step_size,
         bucket_max_pixels=args.bucket_max_pixels,
         dropout=args.tag_dropout,
+        shuffle=args.tag_shuffle,
         template_key=args.train_data_template,
         valid_set_size=args.valid_set_size,
         valid_set_repeat=args.valid_set_repeat,
@@ -860,6 +866,12 @@ def main():
         finally:
             pass
 
+    def on_clip():
+        accelerator.clip_grad_norm_(
+            text_encoder.text_model.embeddings.temp_token_embedding.parameters(),
+            args.max_grad_norm
+        )
+
     loop = partial(
         run_model,
         vae,
@@ -894,8 +906,9 @@ def main():
             loop,
             on_train=on_train,
             on_eval=on_eval,
+            on_clip=on_clip,
         )
-        lr_finder.run(num_epochs=200, end_lr=1e3)
+        lr_finder.run(num_epochs=100, end_lr=1e3)
 
         plt.savefig(basepath.joinpath("lr.png"), dpi=300)
         plt.close()
@@ -975,10 +988,7 @@ def main():
                         accelerator.backward(loss)
 
                         if accelerator.sync_gradients:
-                            accelerator.clip_grad_norm_(
-                                text_encoder.text_model.embeddings.temp_token_embedding.parameters(),
-                                args.max_grad_norm
-                            )
+                            on_clip()
 
                         optimizer.step()
                         if not accelerator.optimizer_step_was_skipped: