llama: tokenizer = LlamaTokenizer.from_pretrained(model_args.model_name_or_path) print_rank_0( "Set the eos_token_id and bos_token_id of LLama model tokenizer", log_file, global_rank, ) tokenizer.eos_token_id = 2 tokenizer.bos_token_id = 1 else: tokenizer = AutoTokenizer.from_pretrai...