From 8364ce697ddf6117fdd4f7222832d546d63880de Mon Sep 17 00:00:00 2001 From: Volpeon Date: Wed, 21 Jun 2023 13:28:49 +0200 Subject: Update --- models/clip/tokenizer.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'models/clip/tokenizer.py') diff --git a/models/clip/tokenizer.py b/models/clip/tokenizer.py index 789b525..a866641 100644 --- a/models/clip/tokenizer.py +++ b/models/clip/tokenizer.py @@ -91,18 +91,21 @@ class MultiCLIPTokenizer(CLIPTokenizer): self.vector_shuffle = shuffle_none def add_multi_tokens( - self, - new_tokens: Union[str, list[str]], - num_vectors: Union[int, list[int]] = 1 + self, new_tokens: Union[str, list[str]], num_vectors: Union[int, list[int]] = 1 ) -> Union[list[int], list[list[int]]]: if isinstance(new_tokens, list): if isinstance(num_vectors, int): num_vectors = [num_vectors] * len(new_tokens) if len(num_vectors) != len(new_tokens): - raise ValueError("Expected new_tokens and num_vectors to have the same len") + raise ValueError( + "Expected new_tokens and num_vectors to have the same len" + ) - return [self.add_multi_tokens(new_token, vecs) for new_token, vecs in zip(new_tokens, num_vectors)] + return [ + self.add_multi_tokens(new_token, vecs) + for new_token, vecs in zip(new_tokens, num_vectors) + ] if isinstance(num_vectors, list): raise ValueError("Expected num_vectors to be int for single token") @@ -129,13 +132,11 @@ class MultiCLIPTokenizer(CLIPTokenizer): return [id] def expand_ids(self, ids: list[int]): - return [ - new_id - for id in ids - for new_id in self.expand_id(id) - ] + return [new_id for id in ids for new_id in self.expand_id(id)] - def expand_batched_ids(self, input_ids: Union[list[int], list[list[int]], tuple[list[int]]]): + def expand_batched_ids( + self, input_ids: Union[list[int], list[list[int]], tuple[list[int]]] + ): if isinstance(input_ids, (list, tuple)) and isinstance(input_ids[0], list): return [self.expand_ids(batch) for batch in input_ids] else: -- cgit v1.2.3-54-g00ecf