1 files changed, 24 insertions, 8 deletions

diff --git a/train_lora.py b/train_lora.py
index dea58cf..167b17a 100644
--- a/train_lora.py
+++ b/train_lora.py
@@ -22,17 +22,19 @@ import transformers
 import numpy as np
 from slugify import slugify
 
-from util.files import load_config, load_embeddings_from_dir
 from data.csv import VlpnDataModule, keyword_filter
+from models.clip.embeddings import patch_managed_embeddings
 from training.functional import train, add_placeholder_tokens, get_models
 from training.strategy.lora import lora_strategy
 from training.optimization import get_scheduler
 from training.sampler import create_named_schedule_sampler
 from training.util import AverageMeter, save_args
+from util.files import load_config, load_embeddings_from_dir
 
 # https://github.com/huggingface/peft/blob/main/examples/lora_dreambooth/train_dreambooth.py
 UNET_TARGET_MODULES = ["to_q", "to_v", "query", "value"]
 TEXT_ENCODER_TARGET_MODULES = ["q_proj", "v_proj"]
+TEXT_ENCODER_TARGET_MODULES_WITH_EMBEDDING = TEXT_ENCODER_TARGET_MODULES + ["token_embedding"]
 
 
 logger = get_logger(__name__)
@@ -44,9 +46,9 @@ torch.backends.cuda.matmul.allow_tf32 = True
 torch.backends.cudnn.benchmark = True
 
 torch._dynamo.config.log_level = logging.WARNING
+torch._dynamo.config.suppress_errors = True
 
 hidet.torch.dynamo_config.use_tensor_core(True)
-hidet.torch.dynamo_config.use_attention(True)
 hidet.torch.dynamo_config.search_space(0)
 
 
@@ -322,6 +324,11 @@ def parse_args():
         help="Bias type for Lora. Can be 'none', 'all' or 'lora_only', only used if use_lora and `train_text_encoder` are True",
     )
     parser.add_argument(
+        "--lora_text_encoder_emb",
+        action="store_true",
+        help="Include token embeddings in training. Prevents usage of TI techniques.",
+    )
+    parser.add_argument(
         "--train_text_encoder_cycles",
         default=999999,
         help="Number of epochs the text encoder will be trained."
@@ -717,12 +724,13 @@ def main():
 
     save_args(output_dir, args)
 
-    tokenizer, text_encoder, vae, unet, noise_scheduler, sample_scheduler, embeddings = get_models(
-        args.pretrained_model_name_or_path,
-        args.emb_alpha,
-        args.emb_dropout
-    )
+    tokenizer, text_encoder, vae, unet, noise_scheduler, sample_scheduler = get_models(args.pretrained_model_name_or_path)
     schedule_sampler = create_named_schedule_sampler(args.schedule_sampler, noise_scheduler.config.num_train_timesteps)
+    
+    def ensure_embeddings():
+        if args.lora_text_encoder_emb:
+            raise ValueError("Can't use TI options when training token embeddings with LoRA")
+        return patch_managed_embeddings(text_encoder, args.emb_alpha, args.emb_dropout)
 
     unet_config = LoraConfig(
         r=args.lora_r,
@@ -736,7 +744,7 @@ def main():
     text_encoder_config = LoraConfig(
         r=args.lora_text_encoder_r,
         lora_alpha=args.lora_text_encoder_alpha,
-        target_modules=TEXT_ENCODER_TARGET_MODULES,
+        target_modules=TEXT_ENCODER_TARGET_MODULES_WITH_EMBEDDING if args.lora_text_encoder_emb else TEXT_ENCODER_TARGET_MODULES,
         lora_dropout=args.lora_text_encoder_dropout,
         bias=args.lora_text_encoder_bias,
     )
@@ -765,6 +773,8 @@ def main():
         unet.enable_gradient_checkpointing()
 
     if len(args.alias_tokens) != 0:
+        embeddings = ensure_embeddings()
+        
         alias_placeholder_tokens = args.alias_tokens[::2]
         alias_initializer_tokens = args.alias_tokens[1::2]
 
@@ -781,6 +791,8 @@ def main():
     placeholder_token_ids = []
 
     if args.embeddings_dir is not None:
+        embeddings = ensure_embeddings()
+        
         embeddings_dir = Path(args.embeddings_dir)
         if not embeddings_dir.exists() or not embeddings_dir.is_dir():
             raise ValueError("--embeddings_dir must point to an existing directory")
@@ -798,6 +810,8 @@ def main():
             embeddings.persist()
 
     if len(args.placeholder_tokens) != 0 and not args.train_dir_embeddings:
+        embeddings = ensure_embeddings()
+        
         placeholder_token_ids, initializer_token_ids = add_placeholder_tokens(
             tokenizer=tokenizer,
             embeddings=embeddings,
@@ -997,6 +1011,8 @@ def main():
     # --------------------------------------------------------------------------------
 
     if args.run_pti and len(placeholder_tokens) != 0:
+        embeddings = ensure_embeddings()
+        
         filter_tokens = [token for token in args.filter_tokens if token in placeholder_tokens]
 
         pti_datamodule = create_datamodule(