diff options
Diffstat (limited to 'models/clip/tokenizer.py')
-rw-r--r-- | models/clip/tokenizer.py | 6 |
1 files changed, 2 insertions, 4 deletions
diff --git a/models/clip/tokenizer.py b/models/clip/tokenizer.py index 63566e0..fbfe790 100644 --- a/models/clip/tokenizer.py +++ b/models/clip/tokenizer.py | |||
@@ -8,7 +8,6 @@ from transformers import CLIPTokenizer | |||
8 | 8 | ||
9 | class MultiCLIPTokenizerItem(NamedTuple): | 9 | class MultiCLIPTokenizerItem(NamedTuple): |
10 | token: str | 10 | token: str |
11 | meta_id: int | ||
12 | ids: list[int] | 11 | ids: list[int] |
13 | 12 | ||
14 | 13 | ||
@@ -38,11 +37,10 @@ class MultiCLIPTokenizer(CLIPTokenizer): | |||
38 | super().add_tokens(multi_token) | 37 | super().add_tokens(multi_token) |
39 | 38 | ||
40 | ids = super().convert_tokens_to_ids(multi_token) | 39 | ids = super().convert_tokens_to_ids(multi_token) |
41 | meta_id = ids[0] | ||
42 | 40 | ||
43 | self.token_map[meta_id] = ids | 41 | self.token_map[ids[0]] = ids |
44 | 42 | ||
45 | return MultiCLIPTokenizerItem(new_tokens, meta_id, ids) | 43 | return MultiCLIPTokenizerItem(new_tokens, ids) |
46 | 44 | ||
47 | def expand_id(self, id: int, vector_shuffle=True): | 45 | def expand_id(self, id: int, vector_shuffle=True): |
48 | if id in self.token_map: | 46 | if id in self.token_map: |