From b42e7fbc29fd8045a2b932eb8ae76587f51f7513 Mon Sep 17 00:00:00 2001 From: Volpeon Date: Sat, 31 Dec 2022 17:12:12 +0100 Subject: Bugfixes for multi-vector token handling --- common.py | 1 - 1 file changed, 1 deletion(-) (limited to 'common.py') diff --git a/common.py b/common.py index 1e7f4b9..691be4e 100644 --- a/common.py +++ b/common.py @@ -30,7 +30,6 @@ def load_embeddings_from_dir(tokenizer: MultiCLIPTokenizer, embeddings: ManagedC if filename.is_file(): with safe_open(filename, framework="pt", device="cpu") as file: embed = file.get_tensor("embed") - added = tokenizer.add_multi_tokens(filename.stem, embed.shape[0]) embeddings.add_embed(added.ids, embed) -- cgit v1.2.3-54-g00ecf