From 56edf85c8b80d49c998bcf26392cce50d552137a Mon Sep 17 00:00:00 2001 From: Volpeon Date: Sat, 31 Dec 2022 23:09:41 +0100 Subject: Update --- models/clip/tokenizer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'models/clip/tokenizer.py') diff --git a/models/clip/tokenizer.py b/models/clip/tokenizer.py index 63566e0..fbfe790 100644 --- a/models/clip/tokenizer.py +++ b/models/clip/tokenizer.py @@ -8,7 +8,6 @@ from transformers import CLIPTokenizer class MultiCLIPTokenizerItem(NamedTuple): token: str - meta_id: int ids: list[int] @@ -38,11 +37,10 @@ class MultiCLIPTokenizer(CLIPTokenizer): super().add_tokens(multi_token) ids = super().convert_tokens_to_ids(multi_token) - meta_id = ids[0] - self.token_map[meta_id] = ids + self.token_map[ids[0]] = ids - return MultiCLIPTokenizerItem(new_tokens, meta_id, ids) + return MultiCLIPTokenizerItem(new_tokens, ids) def expand_id(self, id: int, vector_shuffle=True): if id in self.token_map: -- cgit v1.2.3-54-g00ecf