From 16b92605a59d59c65789c89b54bb97da51908056 Mon Sep 17 00:00:00 2001
From: Volpeon <git@volpeon.ink>
Date: Tue, 21 Feb 2023 09:09:50 +0100
Subject: Embedding normalization: Ignore tensors with grad = 0

---
 models/clip/embeddings.py | 8 --------
 1 file changed, 8 deletions(-)

(limited to 'models')

diff --git a/models/clip/embeddings.py b/models/clip/embeddings.py
index 6c41c33..734730e 100644
--- a/models/clip/embeddings.py
+++ b/models/clip/embeddings.py
@@ -98,14 +98,6 @@ class ManagedCLIPTextEmbeddings(CLIPTextEmbeddings):
 
         return embeds
 
-    def normalize(self, target: float = 0.4, lambda_: float = 1.0):
-        if lambda_ == 0:
-            return
-
-        w = self.temp_token_embedding.weight
-        norm = w[self.temp_token_ids, :].norm(dim=-1, keepdim=True)
-        w[self.temp_token_ids].add_((w[self.temp_token_ids] / norm.clamp_min(1e-12)) * lambda_ * (target - norm))
-
     def forward(
         self,
         input_ids: Optional[torch.LongTensor] = None,
-- 
cgit v1.2.3-70-g09d2