BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", }, ] # make directory "models/tokenizers" if it doesn't exist Expand Down Expand Up @@ -142,8 +145,17 @@ def download_file_with_auth(url, token, save_path): if tokt == TOKENIZER_TYPE.SPM: continue #...