-
Notifications
You must be signed in to change notification settings - Fork 34.4k
Expand file tree
/
Copy pathprocessing_utils.py
More file actions
2393 lines (2107 loc) · 116 KB
/
Copy pathprocessing_utils.py
File metadata and controls
2393 lines (2107 loc) · 116 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Copyright 2022 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Processing saving/loading class for common processors.
"""
import bisect
import copy
import functools
import inspect
import json
import os
import re
import sys
import typing
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Annotated, Any, Literal, TypedDict, TypeVar, Union
import numpy as np
import typing_extensions
from huggingface_hub import is_offline_mode
from huggingface_hub.dataclasses import validate_typed_dict
from huggingface_hub.errors import EntryNotFoundError
from .audio_utils import AudioInput, load_audio, make_list_of_audio
from .dynamic_module_utils import custom_object_save
from .feature_extraction_utils import BatchFeature
from .image_utils import ChannelDimension, ImageInput, is_vision_available, make_flat_list_of_images
from .tokenization_utils_base import (
PaddingStrategy,
PreTokenizedInput,
PreTrainedTokenizerBase,
TextInput,
TruncationStrategy,
)
from .utils import (
AUDIO_TOKENIZER_NAME,
CHAT_TEMPLATE_DIR,
CHAT_TEMPLATE_FILE,
LEGACY_PROCESSOR_CHAT_TEMPLATE_FILE,
PROCESSOR_NAME,
PushToHubMixin,
TensorType,
auto_docstring,
cached_file,
copy_func,
direct_transformers_import,
hf_api,
is_torch_available,
list_repo_templates,
logging,
)
from .utils.chat_template_utils import _get_template_variables, render_jinja_template
from .utils.type_validators import (
device_validator,
image_size_validator,
padding_validator,
positive_any_number,
positive_int,
resampling_validator,
tensor_type_validator,
truncation_validator,
video_metadata_validator,
)
from .video_utils import VideoInput, VideoMetadataType, make_batched_videos
if is_torch_available():
import torch
from .modeling_utils import PreTrainedAudioTokenizerBase
if is_vision_available():
from .image_utils import PILImageResampling
logger = logging.get_logger(__name__)
# type hinting: specifying the type of processor class that inherits from ProcessorMixin
SpecificProcessorType = TypeVar("SpecificProcessorType", bound="ProcessorMixin")
# Dynamically import the Transformers module to grab the attribute classes of the processor from their names.
transformers_module = direct_transformers_import(Path(__file__).parent)
class _LazyAutoProcessorMapping(dict):
"""
Lazy dictionary to avoid circular imports.
The mapping names are only imported when accessed.
"""
_MAPPING_NAMES = {
"image_processor": ("transformers.models.auto.image_processing_auto", "AutoImageProcessor"),
"video_processor": ("transformers.models.auto.video_processing_auto", "AutoVideoProcessor"),
"feature_extractor": ("transformers.models.auto.feature_extraction_auto", "AutoFeatureExtractor"),
"audio_processor": ("transformers.models.auto.feature_extraction_auto", "AutoFeatureExtractor"),
"tokenizer": ("transformers.models.auto.tokenization_auto", "AutoTokenizer"),
}
def __getitem__(self, key):
if key not in self._MAPPING_NAMES:
raise KeyError(key)
module_name, attr_name = self._MAPPING_NAMES[key]
module = __import__(module_name, fromlist=[attr_name])
return getattr(module, attr_name)
def __contains__(self, key):
return key in self._MAPPING_NAMES
def keys(self):
return self._MAPPING_NAMES.keys()
MODALITY_TO_AUTOPROCESSOR_MAPPING = _LazyAutoProcessorMapping()
MODALITY_TO_BASE_CLASS_MAPPING = {
"audio_tokenizer": (
"HiggsAudioV2TokenizerModel",
"DacModel",
), # TODO: @eustlb, to be replaced with PreTrainedAudioTokenizerBase
"audio_processor": "FeatureExtractionMixin",
"tokenizer": ("PreTrainedTokenizerBase", "MistralCommonBackend"),
"feature_extractor": "FeatureExtractionMixin",
"image_processor": "ImageProcessingMixin",
"video_processor": "BaseVideoProcessor",
}
def _get_modality_for_attribute(attribute_name: str) -> str:
"""
Get the canonical modality type for a given attribute name.
For example:
- "image_processor" -> "image_processor"
- "encoder_image_processor" -> "image_processor"
- "text_tokenizer" -> "tokenizer"
- "my_feature_extractor" -> "feature_extractor"
"""
for modality in MODALITY_TO_AUTOPROCESSOR_MAPPING.keys():
if modality in attribute_name:
return modality
raise ValueError(
f"Cannot determine modality for attribute '{attribute_name}'. "
f"Attribute name must contain one of: {list(MODALITY_TO_AUTOPROCESSOR_MAPPING.keys())}"
)
if sys.version_info >= (3, 11):
Unpack = typing.Unpack
else:
Unpack = typing_extensions.Unpack
class TextKwargs(TypedDict, total=False):
"""
Keyword arguments for text processing. For extended documentation, check out tokenization_utils_base methods and
docstrings associated.
Attributes:
add_special_tokens (`bool`, *optional*)
Whether or not to add special tokens when encoding the sequences.
padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*)
Activates and controls padding.
truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*):
Activates and controls truncation.
max_length (`int`, *optional*):
Controls the maximum length to use by one of the truncation/padding parameters.
stride (`int`, *optional*):
If set, the overflowing tokens will contain some tokens from the end of the truncated sequence.
is_split_into_words (`bool`, *optional*):
Whether or not the input is already pre-tokenized.
pad_to_multiple_of (`int`, *optional*):
If set, will pad the sequence to a multiple of the provided value.
return_token_type_ids (`bool`, *optional*):
Whether to return token type IDs.
return_attention_mask (`bool`, *optional*):
Whether to return the attention mask.
return_overflowing_tokens (`bool`, *optional*):
Whether or not to return overflowing token sequences.
return_special_tokens_mask (`bool`, *optional*):
Whether or not to return special tokens mask information.
return_offsets_mapping (`bool`, *optional*):
Whether or not to return `(char_start, char_end)` for each token.
return_length (`bool`, *optional*):
Whether or not to return the lengths of the encoded inputs.
verbose (`bool`, *optional*):
Whether or not to print more information and warnings.
padding_side (`str`, *optional*):
The side on which padding will be applied.
return_mm_token_type_ids (`bool`, *optional*):
Whether to return multimodal token type ids indicating mm placeholder token positions.
return_text_replacement_offsets (`bool`, *optional*):
Whether to return character offsets for each mm placeholder and its replacement.
return_tensors (`str` or [`~utils.TensorType`], *optional*):
If set, will return tensors of a particular framework. Acceptable values are:
- `'pt'`: Return PyTorch `torch.Tensor` objects.
- `'np'`: Return NumPy `np.ndarray` objects.
"""
text_pair: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None
text_target: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None
text_pair_target: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None
add_special_tokens: bool | None
padding: Annotated[bool | str | PaddingStrategy | None, padding_validator()]
truncation: Annotated[bool | str | TruncationStrategy | None, truncation_validator()]
max_length: Annotated[int | None, positive_int()]
stride: Annotated[int | None, positive_int()]
is_split_into_words: bool | None
pad_to_multiple_of: Annotated[int | None, positive_int()]
return_token_type_ids: bool | None
return_attention_mask: bool | None
return_overflowing_tokens: bool | None
return_special_tokens_mask: bool | None
return_offsets_mapping: bool | None
return_length: bool | None
verbose: bool | None
padding_side: Literal["left", "right"] | None
return_mm_token_type_ids: bool | None
return_text_replacement_offsets: bool | None
return_tensors: Annotated[str | TensorType | None, tensor_type_validator()]
class ImagesKwargs(TypedDict, total=False):
"""
Keyword arguments for image processing. For extended documentation, check the appropriate ImageProcessor
class methods and docstrings.
Attributes:
do_convert_rgb (`bool`):
Whether to convert the image to RGB format.
do_resize (`bool`, *optional*):
Whether to resize the image.
size (`dict[str, int]`, *optional*):
Resize the shorter side of the input to `size["shortest_edge"]`.
default_to_square (`bool`, *optional*, defaults to `self.default_to_square`):
Whether to default to a square when resizing, if size is an int.
crop_size (`dict[str, int]`, *optional*):
Desired output size when applying center-cropping.
resample (`PILImageResampling`, *optional*):
Resampling filter to use if resizing the image.
do_rescale (`bool`, *optional*):
Whether to rescale the image by the specified scale `rescale_factor`.
rescale_factor (`int` or `float`, *optional*):
Scale factor to use if rescaling the image.
do_normalize (`bool`, *optional*):
Whether to normalize the image.
image_mean (`float` or `list[float] or tuple[float, float, float]`, *optional*):
Mean to use if normalizing the image.
image_std (`float` or `list[float] or tuple[float, float, float]`, *optional*):
Standard deviation to use if normalizing the image.
do_pad (`bool`, *optional*):
Whether to pad the images in the batch.
pad_size (`dict[str, int]`, *optional*):
The size `{"height": int, "width" int}` to pad the images to.
do_center_crop (`bool`, *optional*):
Whether to center crop the image.
data_format (`ChannelDimension` or `str`, *optional*):
The channel dimension format for the output image.
input_data_format (`ChannelDimension` or `str`, *optional*):
The channel dimension format for the input image.
device (`Union[str, torch.Tensor]`, *optional*):
The device to use for processing (e.g. "cpu", "cuda"), only relevant for torchvision backend.
return_tensors (`str` or [`~utils.TensorType`], *optional*):
If set, will return tensors of a particular framework. Acceptable values are:
- `'pt'`: Return PyTorch `torch.Tensor` objects.
- `'np'`: Return NumPy `np.ndarray` objects.
disable_grouping (`bool`, *optional*):
Whether to group images by shapes when processing or not, only relevant for torchvision backend.
image_seq_length (`int`, *optional*):
The number of image tokens to be used for each image in the input.
Added for backward compatibility but this should be set as a processor attribute in future models.
"""
do_convert_rgb: bool | None
do_resize: bool | None
size: Annotated[int | list[int] | tuple[int, ...] | dict[str, int] | None, image_size_validator()]
default_to_square: bool | None
crop_size: Annotated[int | list[int] | tuple[int, ...] | dict[str, int] | None, image_size_validator()]
resample: Annotated[Union["PILImageResampling", int] | None, resampling_validator()]
do_rescale: bool | None
rescale_factor: float | None
do_normalize: bool | None
image_mean: float | list[float] | tuple[float, ...] | None
image_std: float | list[float] | tuple[float, ...] | None
do_pad: bool | None
pad_size: Annotated[int | list[int] | tuple[int, ...] | dict[str, int] | None, image_size_validator()]
do_center_crop: bool | None
data_format: str | ChannelDimension | None
input_data_format: str | ChannelDimension | None
device: Annotated[Union[str, "torch.device"] | None, device_validator()]
return_tensors: Annotated[str | TensorType | None, tensor_type_validator()]
disable_grouping: bool | None
image_seq_length: int | None
class VideosKwargs(TypedDict, total=False):
"""
Keyword arguments for video processing.
Attributes:
do_convert_rgb (`bool`):
Whether to convert the video to RGB format.
do_resize (`bool`):
Whether to resize the video.
size (`dict[str, int]`, *optional*):
Resize the shorter side of the input to `size["shortest_edge"]`.
default_to_square (`bool`, *optional*, defaults to `self.default_to_square`):
Whether to default to a square when resizing, if size is an int.
resample (`PILImageResampling`, *optional*):
Resampling filter to use if resizing the video.
do_rescale (`bool`, *optional*):
Whether to rescale the video by the specified scale `rescale_factor`.
rescale_factor (`int` or `float`, *optional*):
Scale factor to use if rescaling the video.
do_normalize (`bool`, *optional*):
Whether to normalize the video.
image_mean (`float` or `list[float] or tuple[float, float, float]`, *optional*):
Mean to use if normalizing the video.
image_std (`float` or `list[float] or tuple[float, float, float]`, *optional*):
Standard deviation to use if normalizing the video.
do_center_crop (`bool`, *optional*):
Whether to center crop the video.
do_pad (`bool`, *optional*):
Whether to pad the images in the batch.
do_sample_frames (`bool`, *optional*):
Whether to sample frames from the video before processing or to process the whole video.
video_metadata (`Union[VideoMetadata, dict]`, *optional*):
Metadata of the video containing information about total duration, fps and total number of frames.
num_frames (`int`, *optional*):
Maximum number of frames to sample when `do_sample_frames=True`.
fps (`int` or `float`, *optional*):
Target frames to sample per second when `do_sample_frames=True`.
crop_size (`dict[str, int]`, *optional*):
Desired output size when applying center-cropping.
data_format (`ChannelDimension` or `str`, *optional*):
The channel dimension format for the output video.
input_data_format (`ChannelDimension` or `str`, *optional*):
The channel dimension format for the input video.
device (`Union[str, torch.Tensor]`, *optional*):
The device to use for processing (e.g. "cpu", "cuda"), only relevant for fast image processing.
return_metadata (`bool`, *optional*):
Whether to return video metadata or not.
return_tensors (`str` or [`~utils.TensorType`], *optional*):
If set, will return tensors of a particular framework. Acceptable values are:
- `'pt'`: Return PyTorch `torch.Tensor` objects.
- `'np'`: Return NumPy `np.ndarray` objects.
"""
do_convert_rgb: bool | None
do_resize: bool | None
size: Annotated[int | list[int] | tuple[int, ...] | dict[str, int] | None, image_size_validator()]
default_to_square: bool | None
resample: Annotated[Union["PILImageResampling", int] | None, resampling_validator()]
do_rescale: bool | None
rescale_factor: float | None
do_normalize: bool | None
image_mean: float | list[float] | tuple[float, ...] | None
image_std: float | list[float] | tuple[float, ...] | None
do_center_crop: bool | None
do_pad: bool | None
crop_size: Annotated[int | list[int] | tuple[int, ...] | dict[str, int] | None, image_size_validator()]
data_format: str | ChannelDimension | None
input_data_format: str | ChannelDimension | None
device: Annotated[Union[str, "torch.device"] | None, device_validator()]
do_sample_frames: bool | None
video_metadata: Annotated[VideoMetadataType | None, video_metadata_validator()]
fps: Annotated[int | float | None, positive_any_number()]
num_frames: Annotated[int | None, positive_int()]
return_metadata: bool | None
return_tensors: Annotated[str | TensorType | None, tensor_type_validator()]
class AudioKwargs(TypedDict, total=False):
"""
Keyword arguments for audio processing.
Attributes:
sampling_rate (`int`, *optional*):
The sampling rate at which the `raw_speech` input was sampled.
raw_speech (`np.ndarray`, `list[float]`, `list[np.ndarray]`, `list[list[float]]`):
The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
stereo, i.e. single float per timestep.
padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*):
Select a strategy to pad the returned sequences (according to the model's padding side and padding
index) among:
- `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
sequence if provided).
- `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
acceptable input length for the model if that argument is not provided.
- `False` or `'do_not_pad'`
max_length (`int`, *optional*):
Maximum length of the returned list and optionally padding length (see above).
truncation (`bool`, *optional*):
Activates truncation to cut input sequences longer than *max_length* to *max_length*.
pad_to_multiple_of (`int`, *optional*):
If set, will pad the sequence to a multiple of the provided value.
return_attention_mask (`bool`, *optional*):
Whether or not [`~ASTFeatureExtractor.__call__`] should return `attention_mask`.
return_tensors (`str` or [`~utils.TensorType`], *optional*):
If set, will return tensors of a particular framework. Acceptable values are:
- `'pt'`: Return PyTorch `torch.Tensor` objects.
- `'np'`: Return NumPy `np.ndarray` objects.
load_audio_backend (`str`, *optional*):
Backend used by [`~audio_utils.load_audio`] to decode/resample audio referenced by URL/path
in `apply_chat_template`. One of `"auto"`, `"torchcodec"`, `"librosa"`, `"torchaudio"`.
"""
sampling_rate: Annotated[int | None, positive_int()]
raw_speech: Union["np.ndarray", list[float], list["np.ndarray"], list[list[float]]] | None
padding: Annotated[bool | str | PaddingStrategy | None, padding_validator()]
max_length: Annotated[int | None, positive_int()]
truncation: Annotated[bool | str | TruncationStrategy | None, truncation_validator()]
pad_to_multiple_of: Annotated[int | None, positive_int()]
return_attention_mask: bool | None
return_tensors: Annotated[str | TensorType | None, tensor_type_validator()]
load_audio_backend: str | None
class ProcessingKwargs(TypedDict, total=False):
"""
Base class for kwargs passing to processors.
In case a model has specific kwargs that are not present in the base class or default values for existing keys,
it should have its own `ModelProcessorKwargs` class that inherits from `ProcessingKwargs` to provide:
1) Additional typed keys and that this model requires to process inputs.
2) Default values for existing keys under a `_defaults` attribute.
New keys have to be defined as follows to ensure type hinting is done correctly.
```python
# adding a new image kwarg for this model
class ModelImagesKwargs(ImagesKwargs, total=False):
new_image_kwarg: Optional[bool]
class ModelProcessorKwargs(ProcessingKwargs, total=False):
images_kwargs: ModelImagesKwargs
_defaults = {
"images_kwargs: {
"new_image_kwarg": False,
}
"text_kwargs": {
"padding": "max_length",
},
}
```
For Python 3.8 compatibility, when inheriting from this class and overriding one of the kwargs,
you need to manually update the __annotations__ dictionary. This can be done as follows:
```python
class CustomProcessorKwargs(ProcessingKwargs, total=False):
images_kwargs: CustomImagesKwargs
CustomProcessorKwargs.__annotations__["images_kwargs"] = CustomImagesKwargs # python 3.8 compatibility
```
"""
_defaults = {}
text_kwargs: TextKwargs = {
**TextKwargs.__annotations__,
}
images_kwargs: ImagesKwargs = {
**ImagesKwargs.__annotations__,
}
videos_kwargs: VideosKwargs = {
**VideosKwargs.__annotations__,
}
audio_kwargs: AudioKwargs = {
**AudioKwargs.__annotations__,
}
class TokenizerChatTemplateKwargs(TypedDict, total=False):
"""
NOTE: `TokenizerChatTemplateKwargs` is deprecated and will be removed in future versions
Keyword arguments for tokenizer's `apply_chat_template`, when it is called from within a processor.
tools (`list[Dict]`, *optional*):
A list of tools (callable functions) that will be accessible to the model. If the template does not
support function calling, this argument will have no effect. Each tool should be passed as a JSON Schema,
giving the name, description and argument types for the tool. See our
[chat templating guide](https://huggingface.co/docs/transformers/main/en/chat_templating#automated-function-conversion-for-tool-use)
for more information.
documents (`list[dict[str, str]]`, *optional*):
A list of dicts representing documents that will be accessible to the model if it is performing RAG
(retrieval-augmented generation). If the template does not support RAG, this argument will have no
effect. We recommend that each document should be a dict containing "title" and "text" keys. Please
see the RAG section of the [chat templating guide](https://huggingface.co/docs/transformers/main/en/chat_templating#arguments-for-RAG)
for examples of passing documents with chat templates.
add_generation_prompt (bool, *optional*):
If this is set, a prompt with the token(s) that indicate
the start of an assistant message will be appended to the formatted output. This is useful when you want to generate a response from the model.
Note that this argument will be passed to the chat template, and so it must be supported in the
template for this argument to have any effect.
continue_final_message (bool or str, *optional*):
If this is set, the chat will be formatted so that the final
message in the chat is open-ended, without any EOS tokens. The model will continue this message
rather than starting a new one. This allows you to "prefill" part of
the model's response for it. If a string is passed, it will be used as the key for the field to continue
(e.g. "reasoning_content"). Cannot be used at the same time as `add_generation_prompt`.
return_assistant_tokens_mask (`bool`, defaults to `False`):
Whether to return a mask of the assistant generated tokens. For tokens generated by the assistant,
the mask will contain 1. For user and system tokens, the mask will contain 0.
This functionality is only available for chat templates that support it via the `{% generation %}` keyword.
reasoning_effort (`str`, *optional*):
The reasoning effort level to use for the model's response. Supported values depend on the model
(e.g. `"none"`, "low"`, `"medium"`, `"high"`). If the template does not support reasoning effort,
this argument will have no effect.
"""
tools: list[dict] | None = None
documents: list[dict[str, str]] | None = None
add_generation_prompt: bool | None = False
continue_final_message: bool | str | None = False
return_assistant_tokens_mask: bool | None = False
reasoning_effort: str | None = None
class ProcessorChatTemplateKwargs(TokenizerChatTemplateKwargs, total=False):
"""
NOTE: `ProcessorChatTemplateKwargs` is deprecated and will be removed in future versions
Keyword arguments for processor's `apply_chat_template`.
tokenize (`bool`, *optional*, defaults to `False`):
Whether to tokenize the output or not.
return_dict (`bool`, defaults to `False`):
Whether to return a dictionary with named outputs. Has no effect if tokenize is `False`.
load_audio_from_video (`bool`, *optional*, defaults to `False`):
Whether to use the audio track of input video. If `True` the audio track will be loaded and passed to the
processor. This flag has no effect if the model doesn't support audio modality.
"""
tokenize: bool | None = False
return_dict: bool | None = False
load_audio_from_video: bool | None = False
class AllKwargsForChatTemplate(TypedDict, total=False):
"NOTE: `AllKwargsForChatTemplate` is deprecated and will be removed in future versions"
processor_kwargs: ProcessingKwargs
template_kwargs: ProcessorChatTemplateKwargs
@dataclass
class MultiModalData:
"""
Dataclass that holds extra useful data for processing
multimodal data. Processors currently cannot return keys,
unless it is used in model's forward. Thus we have helper
methods that calculate and return useful data from processing
input multimodals (images/videos).
Note that this dataclass is aimed to be used only in vLLM
and we might change its API in the future.
"""
num_image_tokens: list[int] | None = None
num_video_tokens: list[int] | None = None
num_audio_tokens: list[int] | None = None
num_image_patches: list[int] | None = None
def __contains__(self, key):
return hasattr(self, key) and getattr(self, key) is not None
def __getitem__(self, key):
if hasattr(self, key):
return getattr(self, key)
raise AttributeError(f"{self.__class__.__name__} has no attribute {key}")
@functools.lru_cache(maxsize=8)
def _merge_typed_dict(preprocessor_typed_dict: type, modality_typed_dict: type) -> type:
return TypedDict(
"merged_typed_dict",
{**preprocessor_typed_dict.__annotations__, **modality_typed_dict.__annotations__},
total=False,
)
class ProcessorMixin(PushToHubMixin):
"""
This is a mixin used to provide saving/loading functionality for all processor classes.
"""
# Dynamically set sub-processor attributes. Not every processor has all of these;
# they are populated via setattr in __init__ based on each subclass's `attributes`.
tokenizer: Any
feature_extractor: Any
image_processor: Any
video_processor: Any
chat_template: str | dict[str, str] | None
# Names need to be attr_class for attr in attributes
_auto_class = None
valid_processor_kwargs = ProcessingKwargs
skip_tensor_conversion = ["video_metadata", "text_replacement_offsets"]
# args have to match the attributes class attribute
def __init__(self, *args, **kwargs):
# First, extract chat template from kwargs. It can never be a positional arg
setattr(self, "chat_template", kwargs.pop("chat_template", None))
# Check audio tokenizer for its class but do not treat it as attr to avoid saving weights
if (audio_tokenizer := kwargs.pop("audio_tokenizer", None)) is not None:
proper_class = self.check_argument_for_proper_class("audio_tokenizer", audio_tokenizer)
if not (is_torch_available() and isinstance(audio_tokenizer, PreTrainedAudioTokenizerBase)):
raise ValueError(
f"Tried to use `{proper_class}` for audio tokenization. However, this class is not"
" registered for audio tokenization."
)
setattr(self, "audio_tokenizer", audio_tokenizer)
# Sanitize args and kwargs
for key in kwargs:
if key not in self.get_attributes():
raise TypeError(f"Unexpected keyword argument {key}.")
for arg, attribute_name in zip(args, self.get_attributes()):
if attribute_name in kwargs:
raise TypeError(f"Got multiple values for argument {attribute_name}.")
else:
kwargs[attribute_name] = arg
if len(kwargs) != len(self.get_attributes()):
raise ValueError(
f"This processor requires {len(self.get_attributes())} arguments: {', '.join(self.get_attributes())}. Got "
f"{len(args)} arguments instead."
)
# Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
for attribute_name, arg in kwargs.items():
self.check_argument_for_proper_class(attribute_name, arg)
setattr(self, attribute_name, arg)
@auto_docstring
def __call__(
self,
images: ImageInput | None = None,
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
videos: VideoInput | None = None,
audio: AudioInput | None = None,
**kwargs: Unpack[ProcessingKwargs],
):
images, text, videos, audio = self.prepare_inputs_layout(
images=images, text=text, videos=videos, audio=audio, **kwargs
)
self.validate_inputs(images=images, text=text, videos=videos, audio=audio, **kwargs)
merged_kwargs = self._merge_kwargs(
self.valid_processor_kwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs if hasattr(self, "tokenizer") else {},
**kwargs,
)
processed_images = processed_videos = processed_audio = {}
images_replacements = videos_replacements = audio_replacements = []
if images is not None and hasattr(self, "image_processor"):
processed_images, images_replacements = self._process_images(images, **merged_kwargs["images_kwargs"])
if videos is not None and hasattr(self, "video_processor"):
processed_videos, videos_replacements = self._process_videos(videos, **merged_kwargs["videos_kwargs"])
if audio is not None and self._audio_processor is not None:
processed_audio, audio_replacements = self._process_audio(audio, **merged_kwargs["audio_kwargs"])
text_inputs = {}
return_tensors = merged_kwargs["text_kwargs"].get("return_tensors", None)
if getattr(self, "tokenizer", None) is not None and text is not None:
return_mm_token_type_ids = merged_kwargs["text_kwargs"].pop("return_mm_token_type_ids", False)
return_text_replacement_offsets = merged_kwargs["text_kwargs"].pop(
"return_text_replacement_offsets", False
)
text, text_replacement_offsets = self.get_text_with_replacements(
text,
images_replacements,
videos_replacements,
audio_replacements,
)
text_inputs = self.tokenizer(text, **merged_kwargs["text_kwargs"])
self._check_special_mm_tokens(text, text_inputs, modalities=["image", "video", "audio"])
if return_text_replacement_offsets:
text_inputs["text_replacement_offsets"] = text_replacement_offsets
if return_mm_token_type_ids:
text_inputs["mm_token_type_ids"] = self.create_mm_token_type_ids(text_inputs["input_ids"])
# Pop unused keys from the inputs, e.g. inputs used only to compute number of image tokens
data = {**text_inputs, **processed_images, **processed_videos, **processed_audio}
data = {k: v for k, v in data.items() if k not in self.unused_input_names}
if not kwargs.get("return_metadata"):
data.pop("video_metadata", None)
return BatchFeature(data, tensor_type=return_tensors, skip_tensor_conversion=self.skip_tensor_conversion)
def prepare_inputs_layout(
self,
images: ImageInput | None = None,
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
videos: VideoInput | None = None,
audio: AudioInput | None = None,
**kwargs: Unpack[ProcessingKwargs],
):
"""
Normalize and prefetch inputs before processing. Wraps text in a list for multimodal
processors, fetches remote images and audio if URLs are provided, and ensures audio
is properly batched. Returns the normalized `(images, text, videos, audio)` tuple.
"""
# To support BC with models in pre-MLLM era, don't wrap text in list
if self.all_special_multimodal_tokens and text is not None:
if isinstance(text, str):
text = [text]
# avoid in-place updates on text
text = list(text).copy()
if audio is not None and self._audio_processor is not None:
sampling_rate = kwargs.get("sampling_rate", self._audio_processor.sampling_rate)
audio = self._audio_processor.fetch_audio(audio, sampling_rate=sampling_rate)
audio = make_list_of_audio(audio)
if images is not None and hasattr(self, "image_processor"):
images = self.image_processor.fetch_images(images)
return images, text, videos, audio
def validate_inputs(
self,
images: ImageInput | None = None,
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
videos: VideoInput | None = None,
audio: AudioInput | None = None,
**kwargs: Unpack[ProcessingKwargs],
):
"""
Validate that at least one input is provided and that no deprecated keyword arguments
are used. Raises ``ValueError`` otherwise.
Override when the processor needs additional validation on the input args.
"""
if "audios" in kwargs and audio is None:
raise ValueError("You passed keyword argument `audios` which is deprecated. Please use `audio` instead.")
if images is None and text is None and videos is None and audio is None:
raise ValueError(f"You need to provide at least one input to call {self.__class__.__name__}")
# Simple preprocessing includes calling the `subprocessor` and optionally
# building placeholder strings. Each processor can override and add their
# own special pre/post processing on top, e.g. see `audioflamingo`
def _process_images(self, images: ImageInput, **kwargs):
processed_images = self.image_processor(images, **kwargs)
image_replacements = []
if getattr(self, "image_token", None) is not None:
# Some processors use nested struct, we need to flatten back if needed
images = make_flat_list_of_images(images)
for idx in range(len(images)):
replacement_text = self.replace_image_token(processed_images, image_idx=idx, **kwargs)
image_replacements.append(replacement_text)
return processed_images, image_replacements
def _process_videos(self, videos: VideoInput, **kwargs):
processed_videos = self.video_processor(videos, **kwargs)
video_replacements = []
if getattr(self, "video_token", None) is not None:
videos = make_batched_videos(videos)
for idx in range(len(videos)):
replacement_text = self.replace_video_token(processed_videos, video_idx=idx, **kwargs)
video_replacements.append(replacement_text)
return processed_videos, video_replacements
@property
def _audio_processor(self):
# TODO: To be replaced with `audio_processor`
return getattr(self, "audio_processor", getattr(self, "feature_extractor", None))
def _process_audio(self, audio: AudioInput, **kwargs):
processed_audio = self._audio_processor(audio, **kwargs)
audio_replacements = []
if getattr(self, "audio_token", None) is not None:
for idx in range(len(audio)):
replacement_text = self.replace_audio_token(processed_audio, audio_idx=idx, **kwargs)
audio_replacements.append(replacement_text)
return processed_audio, audio_replacements
# To be overridden by each model's processor if they need to add placeholder tokens
def replace_image_token(self, image_inputs: dict, image_idx: int, **kwargs) -> str:
raise NotImplementedError
def replace_video_token(self, video_inputs: dict, video_idx: int, **kwargs) -> str:
raise NotImplementedError
def replace_audio_token(self, audio_inputs: dict, audio_idx: int, **kwargs) -> str:
raise NotImplementedError
def get_text_with_replacements(
self,
text: list[str],
images_replacements: list[str] = [],
videos_replacements: list[str] = [],
audio_replacements: list[str] = [],
) -> tuple[list[str], list[dict[str, Any]]]:
"""
Replace multimodal placeholder tokens in a batch of text strings with their
expanded representations, and return the modified texts alongside offset metadata.
This method is the core text-side preprocessing step for multimodal inputs. It
scans each text in the batch for special tokens (image, video, audio) and replaces
them in-order with the pre-computed replacement strings produced by
`self.replace_image_token` / `self.replace_video_token` / `self.replace_audio_token`.
Replacements are consumed from each modality's list sequentially, so the i-th
occurrence of e.g. ``self.image_token`` is replaced by ``images_replacements[i]``.
To add a new multimodal processor with placeholder tokens, you need to define a correct
`self.image_token` which is the same token that is embedded in input text and also used as
placeholder and repeated many times. Then you need to override `self.replace_image_token`
to return the correct replacement string for a given image at index `i`. Same goes for all
other supported modalities.
Args:
text (`list[str]`):
Batch of raw text strings, each potentially containing multimodal
placeholder tokens. Note that it will be modified in-place and returned.
images_replacements (`list[str]`, *optional*, defaults to `[]`):
Expanded replacement strings for each image, in the order they appear
across the batch. Produced by `self._process_images`.
videos_replacements (`list[str]`, *optional*, defaults to `[]`):
Expanded replacement strings for each video. Produced by
`self._process_videos`.
audio_replacements (`list[str]`, *optional*, defaults to `[]`):
Expanded replacement strings for each audio input. Produced by
`self._process_audio`.
Returns:
`tuple[list[str], list[dict[str, Any]]]`: A tuple of:
- The modified `text` batch with all placeholder tokens expanded.
- `batch_replacement_offsets`: one entry per batch item, each being a
list of dicts with keys:
- `"type"` (`str`): modality name — `"image"`, `"video"`, or `"audio"`
- `"span"` (`tuple[int, int]`): original `(start, end)` char offsets of the placeholder token
- `"new_span"` (`tuple[int, int]`): `(start, end)` offsets of placeholder in the expanded string
- `"text"` (`str`): the original placeholder token string that was matched
- `"replacement"` (`str`): the string it was replaced with
"""
# Early exit if no special tokens found, nothing to replace
if not self.all_special_multimodal_tokens:
return text, []
# Use named regex so we can extract groups later and replace
# TODO @raushan: vllm encodes text and mm-data separately causing errors when a placeholder
# has no associated mm-data. Thus we can check if there are any `replacements` and skip otherwise
# Plan: update all models and contrib to vllm, they might benefit largely from `replacement_offsets`
token_groups = []
if len(images_replacements) > 0 and (image_token := getattr(self, "image_token", None)) is not None:
token_groups.append(f"(?P<image>{re.escape(image_token)})")
if len(videos_replacements) > 0 and (video_token := getattr(self, "video_token", None)) is not None:
token_groups.append(f"(?P<video>{re.escape(video_token)})")
if len(audio_replacements) > 0 and (audio_token := getattr(self, "audio_token", None)) is not None:
token_groups.append(f"(?P<audio>{re.escape(audio_token)})")
regex_special_mm_tokens = "|".join(token_groups) or r"(?!)"
replacements_iters = {
"image": iter(images_replacements),
"video": iter(videos_replacements),
"audio": iter(audio_replacements),
}
batch_replacement_offsets = []
for batch_idx in range(len(text)):
last = 0
offset = 0
replacement_offsets = []
expanded_sample = []
for m in re.finditer(regex_special_mm_tokens, text[batch_idx]):
start, end = m.span()
expanded_sample.append(text[batch_idx][last:start])
# adjust spans using running offset if one sample has several MM data associated
start_with_offset = start + offset
mm_type = m.lastgroup
replacement_text = next(replacements_iters[mm_type])
replacement_offsets.append(
{
"type": mm_type,
"span": (start, end),
"new_span": (start_with_offset, start_with_offset + len(replacement_text)),
"text": m.group(),
"replacement": replacement_text,
}
)
expanded_sample.append(replacement_text)
# update the offsets and the last position
offset += len(replacement_text) - (end - start)
last = end
expanded_sample.append(text[batch_idx][last:])
text[batch_idx] = "".join(expanded_sample)
batch_replacement_offsets.append(replacement_offsets)
return text, batch_replacement_offsets
def create_mm_token_type_ids(self, input_ids: list) -> list[list[int]]:
"""
Build per-token modality type IDs for a batch of token_id sequences.
Each position is assigned an integer indicating which modality it belongs to:
``0`` for regular text, ``1`` for image tokens, ``2`` for video tokens, and
``3`` for audio tokens. Membership is determined by comparing against
``self.image_token_ids``, ``self.video_token_ids``, and ``self.audio_token_ids``.
Args:
input_ids (`list[list[int]]`):
Batch of token ID sequences. May be unpadded (variable length), so
a plain Python list of lists is expected rather than a tensor or
uniformly-shaped array.
Returns:
`list[list[int]]`: A list of the same structure as ``input_ids``, where each
integer is the modality type ID for the corresponding token.
"""
mm_token_type_ids = []
for tokenizer_input in input_ids:
# Convert tensor rows to a list so `np.array` avoids NumPy 2.0's `__array__` copy-keyword deprecation.
if not isinstance(tokenizer_input, list):
tokenizer_input = tokenizer_input.tolist()
tokenizer_input = np.array(tokenizer_input)
mm_token_types = np.zeros_like(tokenizer_input)
mm_token_types[np.isin(tokenizer_input, self.image_token_ids)] = 1
mm_token_types[np.isin(tokenizer_input, self.video_token_ids)] = 2
mm_token_types[np.isin(tokenizer_input, self.audio_token_ids)] = 3
mm_token_type_ids.append(mm_token_types.tolist())
return mm_token_type_ids
@property
def all_special_multimodal_tokens(self) -> list[str]:
special_mm_tokens = [
getattr(self, f"{modality}_token")
for modality in ["image", "video", "audio"]
if getattr(self, f"{modality}_token", None) is not None
]
return special_mm_tokens
# Special ids used per each modality in multimodal models. Models need to
# override if they use special BOI/EOI/row/col/etc tokens that have to be marked
# These values are used to build `mm_token_type_ids`
@property
def image_token_ids(self) -> list[int | None]:
if _image_token_ids := getattr(self, "_image_token_ids", None):
return _image_token_ids
return [getattr(self, "image_token_id", None)]
@image_token_ids.setter
def image_token_ids(self, value: list[int | None]):
setattr(self, "_image_token_ids", value)
@property
def video_token_ids(self) -> list[int | None]:
if _video_token_ids := getattr(self, "_video_token_ids", None):
return _video_token_ids
return [getattr(self, "video_token_id", None)]
@video_token_ids.setter
def video_token_ids(self, value: list[int | None]):
setattr(self, "_video_token_ids", value)
@property
def audio_token_ids(self) -> list[int | None]:
if _audio_token_ids := getattr(self, "_audio_token_ids", None):
return _audio_token_ids
return [getattr(self, "audio_token_id", None)]
@audio_token_ids.setter
def audio_token_ids(self, value: list[int | None]):
setattr(self, "_audio_token_ids", value)
def check_argument_for_proper_class(self, argument_name, argument):
"""
Checks the passed argument's class against the expected transformers class. In case of an unexpected
mismatch between expected and actual class, an error is raise. Otherwise, the proper retrieved class
is returned.
"""
# If the exact attribute name is not in the mapping, use its canonical modality