Skip to content

NLP

Jump to bottom
franklinic edited this page Jan 19, 2026 · 1 revision

Natural Language Processing

AiDotNet provides comprehensive NLP capabilities for text processing, classification, and generation.

Text Preprocessing

Tokenization

using AiDotNet.NLP;

// Word tokenizer
var tokenizer = new WordTokenizer();
var tokens = tokenizer.Tokenize("Hello, world! This is AiDotNet.");
// ["Hello", ",", "world", "!", "This", "is", "AiDotNet", "."]

// Sentence tokenizer
var sentTokenizer = new SentenceTokenizer();
var sentences = sentTokenizer.Tokenize(text);

// Subword tokenizers
var bpeTokenizer = new BPETokenizer(vocabSize: 30000);
await bpeTokenizer.TrainAsync(corpus);

var wordPieceTokenizer = new WordPieceTokenizer(vocabSize: 30000);
var sentencePieceTokenizer = new SentencePieceTokenizer(modelPath: "tokenizer.model");

Text Cleaning

var cleaner = new TextCleaner()
    .RemoveHtml()
    .RemoveUrls()
    .RemoveEmails()
    .RemoveNumbers()
    .RemovePunctuation()
    .RemoveExtraWhitespace()
    .ToLowercase();

var cleanedText = cleaner.Clean(rawText);

Stop Words

var stopWords = StopWords.English;
var filtered = tokens.Where(t => !stopWords.Contains(t.ToLower()));

// Custom stop words
var customStopWords = new HashSet<string> { "the", "a", "an", "is", "are" };

Stemming and Lemmatization

// Stemming (rule-based)
var stemmer = new PorterStemmer();
var stemmed = stemmer.Stem("running");  // "run"

// Lemmatization (dictionary-based)
var lemmatizer = new WordNetLemmatizer();
var lemma = lemmatizer.Lemmatize("better", PartOfSpeech.Adjective);  // "good"

Text Classification

Sentiment Analysis

var result = await new PredictionModelBuilder<float, string, int>()
    .WithTrainingData(reviews, sentiments)  // 0=negative, 1=positive
    .WithTextClassifier(classifier => classifier
        .UseTransformer("distilbert-base-uncased")
        .MaxLength(512))
    .WithOptimizer(OptimizerType.AdamW, learningRate: 2e-5)
    .WithEpochs(3)
    .BuildAsync();

var sentiment = result.Model.Predict("This product is amazing!");

Multi-class Classification

var result = await new PredictionModelBuilder<float, string, int>()
    .WithTrainingData(documents, categories)
    .WithTextClassifier(classifier => classifier
        .UseTransformer("bert-base-uncased")
        .NumLabels(numCategories))
    .BuildAsync();

Using Pre-trained Models

// Load pre-trained sentiment model
var classifier = await TextClassifier<float>.LoadAsync(
    model: "nlptown/bert-base-multilingual-uncased-sentiment");

var result = classifier.Classify("I love this!");
// { Label: "5 stars", Score: 0.95 }

Named Entity Recognition (NER)

var nerModel = await NERModel<float>.LoadAsync("dslim/bert-base-NER");

var entities = nerModel.Extract("Apple CEO Tim Cook announced the new iPhone in Cupertino.");
// [
//   { Text: "Apple", Label: "ORG", Start: 0, End: 5 },
//   { Text: "Tim Cook", Label: "PER", Start: 10, End: 18 },
//   { Text: "iPhone", Label: "MISC", Start: 37, End: 43 },
//   { Text: "Cupertino", Label: "LOC", Start: 47, End: 56 }
// ]

Training Custom NER

var result = await new PredictionModelBuilder<float, string, NEROutput>()
    .WithTrainingData(texts, annotations)  // BIO format
    .WithNERModel(ner => ner
        .UseTransformer("bert-base-uncased")
        .Labels(new[] { "O", "B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC" }))
    .BuildAsync();

Text Embeddings

Sentence Embeddings

var embedder = new SentenceTransformer<float>("all-MiniLM-L6-v2");

var embedding = await embedder.EncodeAsync("Hello world");
// float[384]

var embeddings = await embedder.EncodeAsync(sentences);
// float[n, 384]

Similarity Search

var query = "What is machine learning?";
var documents = new[] {
    "Machine learning is a subset of AI",
    "The weather is nice today",
    "Deep learning uses neural networks"
};

var queryEmbedding = await embedder.EncodeAsync(query);
var docEmbeddings = await embedder.EncodeAsync(documents);

var similarities = CosineSimilarity.Compute(queryEmbedding, docEmbeddings);
// [0.85, 0.12, 0.78]

Word Embeddings

// Word2Vec
var word2vec = await Word2Vec<float>.LoadAsync("word2vec-google-news-300");
var vector = word2vec.GetVector("king");
var similar = word2vec.MostSimilar("king", topK: 10);

// GloVe
var glove = await GloVe<float>.LoadAsync("glove.6B.300d");

// FastText
var fasttext = await FastText<float>.LoadAsync("fasttext-wiki-news-subwords-300");

Question Answering

Extractive QA

var qa = await QuestionAnswering<float>.LoadAsync("deepset/roberta-base-squad2");

var result = qa.Answer(
    question: "What is the capital of France?",
    context: "France is a country in Europe. Its capital is Paris, which is known for the Eiffel Tower."
);
// { Answer: "Paris", Score: 0.95, Start: 47, End: 52 }

Training Custom QA

var result = await new PredictionModelBuilder<float, QAInput, QAOutput>()
    .WithTrainingData(qaDataset)  // SQuAD format
    .WithQuestionAnswering(qa => qa
        .UseTransformer("bert-base-uncased")
        .MaxQueryLength(64)
        .MaxContextLength(384))
    .BuildAsync();

Text Generation

GPT-style Generation

var generator = await TextGenerator<float>.LoadAsync("gpt2");

var output = generator.Generate(
    prompt: "Once upon a time",
    maxLength: 100,
    temperature: 0.7f,
    topK: 50,
    topP: 0.9f);

Summarization

var summarizer = await Summarizer<float>.LoadAsync("facebook/bart-large-cnn");

var summary = summarizer.Summarize(
    text: longArticle,
    maxLength: 150,
    minLength: 50);

Translation

var translator = await Translator<float>.LoadAsync("Helsinki-NLP/opus-mt-en-de");

var german = translator.Translate("Hello, how are you?");
// "Hallo, wie geht es dir?"

Feature Extraction

TF-IDF

var tfidf = new TfidfVectorizer(
    maxFeatures: 10000,
    minDf: 2,
    maxDf: 0.95f,
    ngramRange: (1, 2));

tfidf.Fit(corpus);
var vectors = tfidf.Transform(documents);

Bag of Words

var bow = new CountVectorizer(maxFeatures: 5000);
bow.Fit(corpus);
var vectors = bow.Transform(documents);

N-grams

var ngrams = NgramExtractor.Extract(text, n: 2);
// ["Hello world", "world how", "how are", "are you"]

Topic Modeling

LDA

var lda = new LatentDirichletAllocation<float>(
    numTopics: 10,
    maxIterations: 100);

lda.Fit(documentVectors);

var topics = lda.GetTopics(topK: 10);
foreach (var topic in topics)
{
    Console.WriteLine($"Topic {topic.Id}: {string.Join(", ", topic.TopWords)}");
}

var docTopics = lda.Transform(document);

BERTopic

var bertopic = new BERTopic<float>(
    embeddingModel: "all-MiniLM-L6-v2",
    umap: new UMAP(nNeighbors: 15, nComponents: 5),
    hdbscan: new HDBSCAN(minClusterSize: 15));

var (topics, probs) = bertopic.FitTransform(documents);

Sequence Labeling

Part-of-Speech Tagging

var posTagger = await POSTagger<float>.LoadAsync("flair/pos-english");

var tags = posTagger.Tag("The quick brown fox jumps over the lazy dog");
// [("The", "DT"), ("quick", "JJ"), ("brown", "JJ"), ("fox", "NN"), ...]

Chunking

var chunker = await Chunker<float>.LoadAsync("flair/chunk-english");

var chunks = chunker.Chunk("The quick brown fox jumps over the lazy dog");
// [("The quick brown fox", "NP"), ("jumps", "VP"), ("over the lazy dog", "PP")]

Spell Checking and Correction

var spellChecker = new SpellChecker(dictionary: "en_US");

var corrections = spellChecker.Correct("Ths is a tset sentense.");
// "This is a test sentence."

var suggestions = spellChecker.Suggest("tset");
// ["test", "set", "text", ...]

Text Metrics

// BLEU score
var bleu = Metrics.BLEU(reference, candidate);

// ROUGE scores
var rouge = Metrics.ROUGE(reference, candidate);
Console.WriteLine($"ROUGE-1: {rouge.Rouge1F1}");
Console.WriteLine($"ROUGE-L: {rouge.RougeLF1}");

// Perplexity
var perplexity = model.CalculatePerplexity(text);

Next Steps

Home

Getting Started

Core Concepts

Reference

Community

Clone this wiki locally