Model tokenizer SDK, requires the modeltokenizer docker image
Get Started
$ dotnet add package ModelTokenizerSdkReadme
Model Tokenizer SDK
Model tokenizer SDK. This SDK uses the modeltokenizer docker image found here (repository for the Docker image is here).
New in v3.0.x
- Remove use of static classes for TokenEstimator
- Added
LoggertoModelTokenizer
Help or Feedback
Need help or have feedback? Please file an issue here!
Simple Example
using ModelTokenizerSdk;
ModelTokenizer tokenizer = new ModelTokenizer(endpoint);
tokenizer.Logger = Console.WriteLine;
bool connected = await tokenizer.ValidateConnectivity();
TokenizationResult result1 = await tokenizer.Tokenize(
"sentence-transformers/all-MiniLM-L6-v2", // model
"this is a very simple sentence", // sentence
null, // Huggingface API key
null, // max chunk length
null, // max tokens per chunk
null, // token overlap
);
/*
{
"text": "The quick brown fox jumped quietly over the lazy dog sitting under the tree",
"sha256": "97f4ebc3817b6b2016e7739dc31970b8a4a8cb5f8f06281cdedb21aa49affb24",
"tokens": [
"the",
"quick",
"brown",
"fox",
"jumped",
"quietly",
"over",
"the",
"lazy",
"dog",
"sitting",
"under",
"the",
"tree"
],
"chunks": [
{
"text": "The quick brown fox jumped quietly over the lazy dog sitting under the tree",
"sha256": "97f4ebc3817b6b2016e7739dc31970b8a4a8cb5f8f06281cdedb21aa49affb24",
"token_count": 14
}
]
}
*/
BatchTokenizationResult result2 = await tokenizer.Tokenize(
"sentence-transformers/all-MiniLM-L6-v2", // model
new List<string> {
"this is a very simple sentence",
"hello, how's your day going today?"
},
null // Huggingface API key
);
/*
{
"results": [
{
"text": "this is a very simple sentence",
"sha256": "32392aa65df45f53e4cc19597482acfa78060871ee9af502cc749f126d98f1c2",
"tokens": [
"this",
"is",
"a",
"very",
"simple",
"sentence"
],
"chunks": [
{
"text": "this is a very simple sentence",
"sha256": "32392aa65df45f53e4cc19597482acfa78060871ee9af502cc749f126d98f1c2",
"token_count": 6
}
]
},
{
"text": "hello, how's your day going today?",
"sha256": "8c09b7181ee47076617ac3fbe935d3a7f59fb53822a1302ab8131066f931f4b4",
"tokens": [
"hello",
",",
"how",
"'",
"s",
"your",
"day",
"going",
"today",
"?"
],
"chunks": [
{
"text": "hello, how's your day going today?",
"sha256": "8c09b7181ee47076617ac3fbe935d3a7f59fb53822a1302ab8131066f931f4b4",
"token_count": 10
}
]
},
{
"text": "The quick brown fox jumped quietly over the lazy dog sitting under the tree",
"sha256": "97f4ebc3817b6b2016e7739dc31970b8a4a8cb5f8f06281cdedb21aa49affb24",
"tokens": [
"the",
"quick",
"brown",
"fox",
"jumped",
"quietly",
"over",
"the",
"lazy",
"dog",
"sitting",
"under",
"the",
"tree"
],
"chunks": [
{
"text": "The quick brown fox jumped quietly over the lazy dog sitting under the tree",
"sha256": "97f4ebc3817b6b2016e7739dc31970b8a4a8cb5f8f06281cdedb21aa49affb24",
"token_count": 14
}
]
}
]
}
*/