pkgname = python-tokenizers
pkgbase = python-tokenizers
pkgver = 0.23.2-1
pkgdesc = Fast State-of-the-Art Tokenizers optimized for Research and Production 
url = https://github.com/huggingface/tokenizers
builddate = 1789003050
packager = @termux-user-repository
size = 7811881
arch = aarch64
license = Apache-2.0
depend = libc++
depend = python
depend = python-pip
depend = python-hf-xet
