semchunk-rs

A port of umarbutler/semchunk into Rust for splitting text into semantically meaningful chunks.

Usage

use semchunk::Chunker;
use rust_tokenizers::tokenizer::{RobertaTokenizer, Tokenizer};

const CHUNK_SIZE: usize = 4;

fn main() {
    let tokenizer = RobertaTokenizer
        ::from_file("data/roberta-base-vocab.json", "data/roberta-base-merges.txt", false, false)
        .unwrap();
    let token_counter = Box::new(move |s: &str| tokenizer.tokenize(s).len());
    let chunker = Chunker::new(4, token_counter);

    let text = "The quick brown fox jumps over the lazy dog.";
    let chunks = chunker.chunk(text);

    for (i, chunk) in chunks.iter().enumerate() {
        println!("{}) {}", i + 1, chunk);
    }
}

1) The quick brown fox
2) jumps over the
3) lazy dog.

Benchmarks 📊

Environment:

gutenberg

Benchmarking against the 18 texts of the Gutenberg corpus which contains 3,001,260 tokens. Code here.