diff options
| author | Volpeon <git@volpeon.ink> | 2023-02-07 20:56:37 +0100 |
|---|---|---|
| committer | Volpeon <git@volpeon.ink> | 2023-02-07 20:56:37 +0100 |
| commit | 757e6af0af2f8de6da696976f3110cd70085adad (patch) | |
| tree | cf219fffa359440bc8f2a2d6dd4a647715d66893 /training | |
| parent | Add Lora (diff) | |
| download | textual-inversion-diff-757e6af0af2f8de6da696976f3110cd70085adad.tar.gz textual-inversion-diff-757e6af0af2f8de6da696976f3110cd70085adad.tar.bz2 textual-inversion-diff-757e6af0af2f8de6da696976f3110cd70085adad.zip | |
Fix Lora memory usage
Diffstat (limited to 'training')
| -rw-r--r-- | training/functional.py | 4 | ||||
| -rw-r--r-- | training/strategy/dreambooth.py | 2 | ||||
| -rw-r--r-- | training/strategy/lora.py | 4 | ||||
| -rw-r--r-- | training/strategy/ti.py | 2 |
4 files changed, 3 insertions, 9 deletions
diff --git a/training/functional.py b/training/functional.py index 8f47734..ccbb4ad 100644 --- a/training/functional.py +++ b/training/functional.py | |||
| @@ -261,8 +261,8 @@ def loss_step( | |||
| 261 | eval: bool = False | 261 | eval: bool = False |
| 262 | ): | 262 | ): |
| 263 | # Convert images to latent space | 263 | # Convert images to latent space |
| 264 | latents = vae.encode(batch["pixel_values"]).latent_dist.sample().detach() | 264 | latents = vae.encode(batch["pixel_values"]).latent_dist.sample() |
| 265 | latents = latents * 0.18215 | 265 | latents = latents * vae.config.scaling_factor |
| 266 | 266 | ||
| 267 | generator = torch.Generator(device=latents.device).manual_seed(seed + step) if eval else None | 267 | generator = torch.Generator(device=latents.device).manual_seed(seed + step) if eval else None |
| 268 | 268 | ||
diff --git a/training/strategy/dreambooth.py b/training/strategy/dreambooth.py index b4c77f3..8aaed3a 100644 --- a/training/strategy/dreambooth.py +++ b/training/strategy/dreambooth.py | |||
| @@ -1,4 +1,3 @@ | |||
| 1 | from contextlib import nullcontext | ||
| 2 | from typing import Optional | 1 | from typing import Optional |
| 3 | from functools import partial | 2 | from functools import partial |
| 4 | from contextlib import contextmanager, nullcontext | 3 | from contextlib import contextmanager, nullcontext |
| @@ -6,7 +5,6 @@ from pathlib import Path | |||
| 6 | import itertools | 5 | import itertools |
| 7 | 6 | ||
| 8 | import torch | 7 | import torch |
| 9 | import torch.nn as nn | ||
| 10 | from torch.utils.data import DataLoader | 8 | from torch.utils.data import DataLoader |
| 11 | 9 | ||
| 12 | from accelerate import Accelerator | 10 | from accelerate import Accelerator |
diff --git a/training/strategy/lora.py b/training/strategy/lora.py index 88d1824..92abaa6 100644 --- a/training/strategy/lora.py +++ b/training/strategy/lora.py | |||
| @@ -1,11 +1,9 @@ | |||
| 1 | from contextlib import nullcontext | ||
| 2 | from typing import Optional | 1 | from typing import Optional |
| 3 | from functools import partial | 2 | from functools import partial |
| 4 | from contextlib import contextmanager, nullcontext | 3 | from contextlib import contextmanager |
| 5 | from pathlib import Path | 4 | from pathlib import Path |
| 6 | 5 | ||
| 7 | import torch | 6 | import torch |
| 8 | import torch.nn as nn | ||
| 9 | from torch.utils.data import DataLoader | 7 | from torch.utils.data import DataLoader |
| 10 | 8 | ||
| 11 | from accelerate import Accelerator | 9 | from accelerate import Accelerator |
diff --git a/training/strategy/ti.py b/training/strategy/ti.py index d306f18..da2b81c 100644 --- a/training/strategy/ti.py +++ b/training/strategy/ti.py | |||
| @@ -1,11 +1,9 @@ | |||
| 1 | from contextlib import nullcontext | ||
| 2 | from typing import Optional | 1 | from typing import Optional |
| 3 | from functools import partial | 2 | from functools import partial |
| 4 | from contextlib import contextmanager, nullcontext | 3 | from contextlib import contextmanager, nullcontext |
| 5 | from pathlib import Path | 4 | from pathlib import Path |
| 6 | 5 | ||
| 7 | import torch | 6 | import torch |
| 8 | import torch.nn as nn | ||
| 9 | from torch.utils.data import DataLoader | 7 | from torch.utils.data import DataLoader |
| 10 | 8 | ||
| 11 | from accelerate import Accelerator | 9 | from accelerate import Accelerator |