tokenizer_mode: str = "auto" download_dir: Optional[str] = None use_np_weights: bool = False use_dummy_weights: bool = False @@ -42,6 +43,12 @@ def add_cli_args( help='name or path of the huggingface model to use') parser.add_argument('--tokenizer', type=str, default=Engine...
default="auto", choices=['auto', 'slow', 'mistral'], help='The tokenizer mode.\n\n* "auto" will use the ' 'fast tokenizer if available.\n* "slow" will ' 'always use the slow tokenizer. \n* ' '"mistral" will always use the `mistral_common` tokenizer.') args = parser.parse...
Add ready label to the PR Enable auto-merge. 🚀 mergify bot added the ci/build label Dec 10, 2024 mgoin reviewed Dec 10, 2024 View reviewed changes Collaborator mgoin left a comment Choose a reason for hiding this comment The reason will be displayed to describe this comment to ot...
default="auto", choices=['auto', 'slow', 'mistral'], help='The tokenizer mode.\n\n* "auto" will use the ' 'fast tokenizer if available.\n* "slow" will ' 'always use the slow tokenizer. \n* ' '"mistral" will always use the `mistral_common` tokenizer.') args = parser.parse...