diff options
Diffstat (limited to 'training')
-rw-r--r-- | training/functional.py | 4 | ||||
-rw-r--r-- | training/strategy/dreambooth.py | 10 | ||||
-rw-r--r-- | training/strategy/ti.py | 13 |
3 files changed, 15 insertions, 12 deletions
diff --git a/training/functional.py b/training/functional.py index 1548784..3d27380 100644 --- a/training/functional.py +++ b/training/functional.py | |||
@@ -87,8 +87,6 @@ def save_samples( | |||
87 | ): | 87 | ): |
88 | print(f"Saving samples for step {step}...") | 88 | print(f"Saving samples for step {step}...") |
89 | 89 | ||
90 | samples_path = output_dir.joinpath("samples") | ||
91 | |||
92 | grid_cols = min(batch_size, 4) | 90 | grid_cols = min(batch_size, 4) |
93 | grid_rows = (num_batches * batch_size) // grid_cols | 91 | grid_rows = (num_batches * batch_size) // grid_cols |
94 | 92 | ||
@@ -120,7 +118,7 @@ def save_samples( | |||
120 | 118 | ||
121 | for pool, data, gen in datasets: | 119 | for pool, data, gen in datasets: |
122 | all_samples = [] | 120 | all_samples = [] |
123 | file_path = samples_path.joinpath(pool, f"step_{step}.jpg") | 121 | file_path = output_dir.joinpath(pool, f"step_{step}.jpg") |
124 | file_path.parent.mkdir(parents=True, exist_ok=True) | 122 | file_path.parent.mkdir(parents=True, exist_ok=True) |
125 | 123 | ||
126 | batches = list(itertools.islice(itertools.cycle(data), batch_size * num_batches)) | 124 | batches = list(itertools.islice(itertools.cycle(data), batch_size * num_batches)) |
diff --git a/training/strategy/dreambooth.py b/training/strategy/dreambooth.py index aeaa828..93c81cb 100644 --- a/training/strategy/dreambooth.py +++ b/training/strategy/dreambooth.py | |||
@@ -27,7 +27,8 @@ def dreambooth_strategy( | |||
27 | sample_scheduler: DPMSolverMultistepScheduler, | 27 | sample_scheduler: DPMSolverMultistepScheduler, |
28 | train_dataloader: DataLoader, | 28 | train_dataloader: DataLoader, |
29 | val_dataloader: Optional[DataLoader], | 29 | val_dataloader: Optional[DataLoader], |
30 | output_dir: Path, | 30 | sample_output_dir: Path, |
31 | checkpoint_output_dir: Path, | ||
31 | seed: int, | 32 | seed: int, |
32 | train_text_encoder_epochs: int, | 33 | train_text_encoder_epochs: int, |
33 | max_grad_norm: float = 1.0, | 34 | max_grad_norm: float = 1.0, |
@@ -47,6 +48,9 @@ def dreambooth_strategy( | |||
47 | "Please set gradient_accumulation_steps to 1. This feature will be supported in the future." | 48 | "Please set gradient_accumulation_steps to 1. This feature will be supported in the future." |
48 | ) | 49 | ) |
49 | 50 | ||
51 | sample_output_dir.mkdir(parents=True, exist_ok=True) | ||
52 | checkpoint_output_dir.mkdir(parents=True, exist_ok=True) | ||
53 | |||
50 | weight_dtype = torch.float32 | 54 | weight_dtype = torch.float32 |
51 | if accelerator.state.mixed_precision == "fp16": | 55 | if accelerator.state.mixed_precision == "fp16": |
52 | weight_dtype = torch.float16 | 56 | weight_dtype = torch.float16 |
@@ -64,7 +68,7 @@ def dreambooth_strategy( | |||
64 | train_dataloader=train_dataloader, | 68 | train_dataloader=train_dataloader, |
65 | val_dataloader=val_dataloader, | 69 | val_dataloader=val_dataloader, |
66 | dtype=weight_dtype, | 70 | dtype=weight_dtype, |
67 | output_dir=output_dir, | 71 | output_dir=sample_output_dir, |
68 | seed=seed, | 72 | seed=seed, |
69 | batch_size=sample_batch_size, | 73 | batch_size=sample_batch_size, |
70 | num_batches=sample_num_batches, | 74 | num_batches=sample_num_batches, |
@@ -156,7 +160,7 @@ def dreambooth_strategy( | |||
156 | tokenizer=tokenizer, | 160 | tokenizer=tokenizer, |
157 | scheduler=sample_scheduler, | 161 | scheduler=sample_scheduler, |
158 | ) | 162 | ) |
159 | pipeline.save_pretrained(output_dir.joinpath("model")) | 163 | pipeline.save_pretrained(checkpoint_output_dir) |
160 | 164 | ||
161 | del unet_ | 165 | del unet_ |
162 | del text_encoder_ | 166 | del text_encoder_ |
diff --git a/training/strategy/ti.py b/training/strategy/ti.py index 9d39e15..00f3529 100644 --- a/training/strategy/ti.py +++ b/training/strategy/ti.py | |||
@@ -27,7 +27,8 @@ def textual_inversion_strategy( | |||
27 | sample_scheduler: DPMSolverMultistepScheduler, | 27 | sample_scheduler: DPMSolverMultistepScheduler, |
28 | train_dataloader: DataLoader, | 28 | train_dataloader: DataLoader, |
29 | val_dataloader: Optional[DataLoader], | 29 | val_dataloader: Optional[DataLoader], |
30 | output_dir: Path, | 30 | sample_output_dir: Path, |
31 | checkpoint_output_dir: Path, | ||
31 | seed: int, | 32 | seed: int, |
32 | placeholder_tokens: list[str], | 33 | placeholder_tokens: list[str], |
33 | placeholder_token_ids: list[list[int]], | 34 | placeholder_token_ids: list[list[int]], |
@@ -47,6 +48,9 @@ def textual_inversion_strategy( | |||
47 | sample_guidance_scale: float = 7.5, | 48 | sample_guidance_scale: float = 7.5, |
48 | sample_image_size: Optional[int] = None, | 49 | sample_image_size: Optional[int] = None, |
49 | ): | 50 | ): |
51 | sample_output_dir.mkdir(parents=True, exist_ok=True) | ||
52 | checkpoint_output_dir.mkdir(parents=True, exist_ok=True) | ||
53 | |||
50 | weight_dtype = torch.float32 | 54 | weight_dtype = torch.float32 |
51 | if accelerator.state.mixed_precision == "fp16": | 55 | if accelerator.state.mixed_precision == "fp16": |
52 | weight_dtype = torch.float16 | 56 | weight_dtype = torch.float16 |
@@ -64,7 +68,7 @@ def textual_inversion_strategy( | |||
64 | train_dataloader=train_dataloader, | 68 | train_dataloader=train_dataloader, |
65 | val_dataloader=val_dataloader, | 69 | val_dataloader=val_dataloader, |
66 | dtype=weight_dtype, | 70 | dtype=weight_dtype, |
67 | output_dir=output_dir, | 71 | output_dir=sample_output_dir, |
68 | seed=seed, | 72 | seed=seed, |
69 | batch_size=sample_batch_size, | 73 | batch_size=sample_batch_size, |
70 | num_batches=sample_num_batches, | 74 | num_batches=sample_num_batches, |
@@ -135,14 +139,11 @@ def textual_inversion_strategy( | |||
135 | def on_checkpoint(step, postfix): | 139 | def on_checkpoint(step, postfix): |
136 | print(f"Saving checkpoint for step {step}...") | 140 | print(f"Saving checkpoint for step {step}...") |
137 | 141 | ||
138 | checkpoints_path = output_dir.joinpath("checkpoints") | ||
139 | checkpoints_path.mkdir(parents=True, exist_ok=True) | ||
140 | |||
141 | with ema_context(): | 142 | with ema_context(): |
142 | for (token, ids) in zip(placeholder_tokens, placeholder_token_ids): | 143 | for (token, ids) in zip(placeholder_tokens, placeholder_token_ids): |
143 | text_encoder.text_model.embeddings.save_embed( | 144 | text_encoder.text_model.embeddings.save_embed( |
144 | ids, | 145 | ids, |
145 | checkpoints_path.joinpath(f"{slugify(token)}_{step}_{postfix}.bin") | 146 | checkpoint_output_dir.joinpath(f"{slugify(token)}_{step}_{postfix}.bin") |
146 | ) | 147 | ) |
147 | 148 | ||
148 | @torch.no_grad() | 149 | @torch.no_grad() |