Text Mining and Statistical NLP

Text mining extracts structured statistical information from unstructured text. The central challenge is converting variable-length text sequences into fixed-dimensional numerical representations amenable to statistical analysis. This lesson develops the mathematical foundations of text representation, topic modeling, sentiment analysis, word embeddings, and document classification.

Bag-of-Words Representation

Topic Models

Sentiment Analysis

Word Embeddings

Document Classification with Naive Bayes

Evaluation Metrics

Python Implementation

import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation, NMF
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report, roc_auc_score
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

np.random.seed(42)

# --- Sample corpus ---
corpus = [
    "The stock market rallied on strong earnings reports",
    "New vaccine shows promising results in clinical trials",
    "Federal Reserve raises interest rates to combat inflation",
    "Tech companies report record profits amid AI boom",
    "Hospital implements new patient safety protocols",
    "Bond yields decline as investors seek safe assets",
    "Study finds link between exercise and mental health",
    "GDP growth exceeds expectations in third quarter",
]
labels = np.array([0, 1, 0, 0, 1, 0, 1, 0])  # 0=finance, 1=health

# --- TF-IDF ---
tfidf = TfidfVectorizer(max_features=1000, stop_words='english')
X_tfidf = tfidf.fit_transform(corpus)
print("TF-IDF shape:", X_tfidf.shape)
print("Top features:", tfidf.get_feature_names_out()[:10])

# --- LDA Topic Model ---
count_vec = CountVectorizer(max_features=1000, stop_words='english')
X_counts = count_vec.fit_transform(corpus)

lda = LatentDirichletAllocation(n_components=2, random_state=42, max_iter=50)
doc_topics = lda.fit_transform(X_counts)

print("\n=== LDA Topics ===")
for idx, topic in enumerate(lda.components_):
    top_words = [count_vec.get_feature_names_out()[i] for i in topic.argsort()[-8:][::-1]]
    print(f"Topic {idx}: {', '.join(top_words)}")

# --- Naive Bayes Classification ---
X_train, X_test, y_train, y_test = train_test_split(
    X_tfidf, labels, test_size=0.25, random_state=42, stratify=labels
)

nb = MultinomialNB(alpha=1.0)
nb.fit(X_train, y_train)
y_pred_nb = nb.predict(X_test)
y_prob_nb = nb.predict_proba(X_test)[:, 1]

print("\n=== Naive Bayes ===")
print(classification_report(y_test, y_pred_nb, target_names=['Finance', 'Health']))

# --- Logistic Regression ---
lr = LogisticRegression(C=1.0, max_iter=1000, random_state=42)
lr.fit(X_train, y_train)
y_pred_lr = lr.predict(X_test)
y_prob_lr = lr.predict_proba(X_test)[:, 1]

print("=== Logistic Regression ===")
print(classification_report(y_test, y_pred_lr, target_names=['Finance', 'Health']))

# --- Word Embeddings (pretrained) ---
# Using synthetic embeddings for demonstration
vocab = tfidf.get_feature_names_out()
k = 50  # embedding dimension
embeddings = np.random.randn(len(vocab), k) * 0.1

# Document embedding = TF-IDF weighted average of word embeddings
def doc_embedding(tfidf_matrix, embeddings):
    doc_embs = tfidf_matrix @ embeddings
    norms = np.linalg.norm(doc_embs, axis=1, keepdims=True)
    norms[norms == 0] = 1
    return doc_embs / norms

X_emb = doc_embedding(X_tfidf, embeddings)
print(f"\nDocument embedding shape: {X_emb.shape}")
print(f"Similarity matrix (cosine):\n{X_emb @ X_emb.T:.3f}")

Summary