From 6c64f769043c8212b1a5778e857af691a828798d Mon Sep 17 00:00:00 2001 From: Volpeon Date: Thu, 5 Jan 2023 10:19:38 +0100 Subject: Various cleanups --- models/clip/tokenizer.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'models/clip/tokenizer.py') diff --git a/models/clip/tokenizer.py b/models/clip/tokenizer.py index 4e97ab5..034adf9 100644 --- a/models/clip/tokenizer.py +++ b/models/clip/tokenizer.py @@ -55,11 +55,6 @@ def shuffle_auto(tokens: list[int]): return shuffle_all(tokens) -class MultiCLIPTokenizerItem(NamedTuple): - token: str - ids: list[int] - - class MultiCLIPTokenizer(CLIPTokenizer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -96,7 +91,7 @@ class MultiCLIPTokenizer(CLIPTokenizer): self, new_tokens: Union[str, list[str]], num_vectors: Union[int, list[int]] = 1 - ) -> Union[MultiCLIPTokenizerItem, list[MultiCLIPTokenizerItem]]: + ) -> Union[list[int], list[list[int]]]: if isinstance(new_tokens, list): if isinstance(num_vectors, int): num_vectors = [num_vectors] * len(new_tokens) @@ -119,7 +114,7 @@ class MultiCLIPTokenizer(CLIPTokenizer): self.token_map[ids[0]] = ids - return MultiCLIPTokenizerItem(new_tokens, ids) + return ids def expand_id(self, id: int): if id in self.token_map: -- cgit v1.2.3-54-g00ecf