ollama pull llama3      # chat model (~4.7 GB)
ollama pull nomic-embed-text   # embedding model (~270 MB, used in §2.7)


# !pip install ollama
import ollama
import json, time, random, re
import pandas as pd
import numpy as np

DEFAULT_MODEL = "llama3"
EMBED_MODEL   = "nomic-embed-text"

# Predefined function to call llama3 (or whichever model you switch to)
def chat(prompt, system=None, model=DEFAULT_MODEL, temperature=0.0, fmt=None):
    messages = []
    if system:
        messages.append({"role": "system", "content": system})
    messages.append({"role": "user", "content": prompt})
    opts = {"temperature": temperature}
    kwargs = dict(model=model, messages=messages, options=opts)
    if fmt == "json":
        kwargs["format"] = "json"
    return ollama.chat(**kwargs)["message"]["content"]

def load_api_key(filepath):
    with open(filepath) as f:
        return f.read().strip()

def check_ollama(chat_model=DEFAULT_MODEL, embed_model=EMBED_MODEL):
    try:
        listed = ollama.list().get("models", [])
        names = [m.get("name", m.get("model", "")) for m in listed]
        bases = {n.split(":")[0] for n in names}
        chat_ok  = chat_model  in bases or any(chat_model  in n for n in names)
        embed_ok = embed_model in bases or any(embed_model in n for n in names)
        print(f"Ollama daemon : OK  ({len(names)} model(s) installed)")
        print(f"  chat  '{chat_model}' : {'FOUND' if chat_ok else 'MISSING -> run: ollama pull ' + chat_model}")
        print(f"  embed '{embed_model}' : {'FOUND' if embed_ok else 'MISSING -> run: ollama pull ' + embed_model}")
        if chat_ok:
            t0 = time.time()
            r = ollama.chat(
                model=chat_model,
                messages=[{"role": "user", "content": "Reply with the single word: pong"}],
                options={"temperature": 0},
            )
            print(f"  ping reply    : '{r['message']['content'].strip()[:40]}'  ({time.time()-t0:.2f}s)")
        print("\nready.")
    except Exception as e:
        print(f"[!] Ollama not reachable: {e}")
        print("    Install + start Ollama from https://ollama.com/, then run:")
        print("      ollama pull llama3")
        print("      ollama pull nomic-embed-text")

check_ollama()

Ollama daemon : OK  (3 model(s) installed)
  chat  'llama3' : FOUND
  embed 'nomic-embed-text' : FOUND
  ping reply    : 'Pong'  (7.27s)

ready.


TRY_PROMPT = (
    "Act as a financial analyst. Based on the following metrics, provide a one-sentence "
    "conclusion starting with 'Signal: [Good/Bad/Neutral]' followed by the short reason why. "
    "Net sales increased 14% to $213.4 billion in the fourth quarter, compared with $187.8 billion in fourth quarter 2024.  "
    "Operating income increased to $25.0 billion in the fourth quarter, compared with $21.2 billion in fourth quarter 2024. "
    "Net income increased to $21.2 billion in the fourth quarter, or $1.95 per diluted share, compared with $20.0 billion, or $1.86 per diluted share, in fourth quarter 2024."
)


# !pip install openai   |   signup via: https://platform.openai.com
try:
    from openai import OpenAI
    client = OpenAI(api_key=load_api_key("key/openai_key.txt"))
    r = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": TRY_PROMPT}],
        seed=42,
    )
    print(r.choices[0].message.content)
except FileNotFoundError:
    print("[skip] no openai_key.txt — sign up at platform.openai.com")
except Exception as e:
    print(f"[skip] {e}")

Signal: Good, as all key financial metrics—net sales, operating income, and net income—show significant improvements year-over-year, indicating a strong performance.


# !pip install anthropic   |  signup via: https://console.anthropic.com
try:
    from anthropic import Anthropic
    client = Anthropic(api_key=load_api_key("key/anthropic_key.txt"))
    msg = client.messages.create(
        model="claude-haiku-4-5",
        max_tokens=200,
        messages=[{"role": "user", "content": TRY_PROMPT}],
    )
    print(msg.content[0].text)
except FileNotFoundError:
    print("[skip] no anthropic_key.txt — sign up at console.anthropic.com")
except Exception as e:
    print(f"[skip] {e}")

Signal: **Good** – Strong double-digit revenue growth of 14% combined with operating income increasing 18% and net income rising 6% with improved EPS demonstrates robust profitability expansion and operational efficiency gains.


# !pip install google-genai   |  signup via: https://aistudio.google.com/apikey (free to get API key)
try:
    from google import genai
    client = genai.Client(api_key=load_api_key("key/google_key.txt"))
    r = client.models.generate_content(
        model="gemini-2.5-flash",
        contents=TRY_PROMPT
    )
    print(r.text)
except FileNotFoundError:
    print("[skip] no google_key.txt — free signup at aistudio.google.com/apikey")
except Exception as e:
    print(f"[skip] {e}")

# Can sometimes fail due to high traffic.

Signal: Good, driven by substantial increases in net sales, operating income, and net income across the board.


print(chat(TRY_PROMPT))

Signal: Good

The strong increase in net sales, operating income, and net income indicates a positive trend for the company, suggesting that its business is performing well and generating significant revenue and profitability growth.


job_post = """Avery Dennison is looking for a Materials Scientist - Machine Learning.
Requirements: Ph.D. in Polymer Science, Chemical Engineering, or Physics. Experience with
Python, TensorFlow, PyTorch, C++. Familiarity with Molecular Dynamics and Finite Element
Analysis. Location: Mentor, OH. Contact: xxx@stevens.edu"""

out = chat(
    prompt=f"Extract entities from this job post. Return JSON with keys: company, job_title, skills (list), location, email.\n\nPOST:\n{job_post}",
    system="You are an information-extraction assistant. Output ONLY valid JSON.",
    fmt="json",
)
print(out)
parsed = json.loads(out)
parsed

{"company": "Avery Dennison", "job_title": "Materials Scientist - Machine Learning", "skills": ["Python", "TensorFlow", "PyTorch", "C++"], "location": "Mentor, OH", "email": "xxx@stevens.edu"}

{'company': 'Avery Dennison',
 'job_title': 'Materials Scientist - Machine Learning',
 'skills': ['Python', 'TensorFlow', 'PyTorch', 'C++'],
 'location': 'Mentor, OH',
 'email': 'xxx@stevens.edu'}


# Strict prompt — may prevent hallucination
hook_passage = (
    "Apple's long-serving CEO delivered the keynote at the company's annual developer "
    "conference this week. The event was held at Apple's California headquarters campus "
    "and drew thousands of developers from around the world. Management also highlighted "
    "continued growth in services revenue during the most recent quarter and previewed "
    "new AI capabilities."
)

strict_prompt = (
    "Complete the company briefing below. Output as JSON with these exact fields:\n"
    "- company_name\n"
    "- ceo_full_name\n"
    "- headquarters_city\n"
    "- most_recent_quarter_services_revenue_usd_billion\n"
    "- total_fy2023_revenue_usd_billion\n"
    "- approximate_worldwide_developer_attendance\n\n"
    "IMPORTANT: If a field is not explicitly stated in the passage, return null. "
    "Do NOT use outside knowledge. Do NOT guess.\n\n"
    f"PASSAGE:\n{hook_passage}"
)

out = chat(
    prompt=strict_prompt,
    system="You are an information-extraction assistant. Output ONLY valid JSON.",
    fmt="json",
)
print(out)
parsed = json.loads(out)
parsed

{
"company_name": "Apple",
"ceo_full_name": null,
"headquarters_city": "California",
"most_recent_quarter_services_revenue_usd_billion": null,
"total_fy2023_revenue_usd_billion": null,
"approximate_worldwide_developer_attendance": "thousands"
}

{'company_name': 'Apple',
 'ceo_full_name': None,
 'headquarters_city': 'California',
 'most_recent_quarter_services_revenue_usd_billion': None,
 'total_fy2023_revenue_usd_billion': None,
 'approximate_worldwide_developer_attendance': 'thousands'}


# LOOSER prompt — no "null if not stated", and a "fill in every field" framing
# that primes the model to complete rather than refuse.
loose_prompt = (
    "Complete the company briefing below. Fill in every field. "
    "Output as JSON with these exact fields:\n"
    "- company_name\n"
    "- ceo_full_name\n"
    "- headquarters_city\n"
    "- most_recent_quarter_services_revenue_usd_billion\n"
    "- total_fy2023_revenue_usd_billion\n"
    "- year_current_campus_opened\n"
    "- approximate_worldwide_developer_attendance\n\n"
    f"PASSAGE:\n{hook_passage}"
)
loose_system = "You are an information-extraction assistant. Output ONLY valid JSON."

print("=== llama3 (local, loose prompt) ===")
print(chat(prompt=loose_prompt, system=loose_system, fmt="json"))

print("\n=== claude-sonnet-4-5 (API, loose prompt) ===")
try:
    from anthropic import Anthropic
    c = Anthropic(api_key=load_api_key("key/anthropic_key.txt"))
    msg = c.messages.create(
        model="claude-sonnet-4-5",
        max_tokens=400,
        system=loose_system,
        messages=[{"role":"user","content":loose_prompt}],
    )
    print(msg.content[0].text)
except FileNotFoundError:
    print("[skip] no anthropic_key.txt — sign up at console.anthropic.com")
except Exception as e:
    print(f"[skip] {e}")

=== llama3 (local, loose prompt) ===
{
"company_name": "Apple",
"ceo_full_name": "CEO (first name not mentioned)",
"headquarters_city": "California",
"most_recent_quarter_services_revenue_usd_billion": null,
"total_fy2023_revenue_usd_billion": null,
"year_current_campus_opened": null,
"approximate_worldwide_developer_attendance": "thousands"
}

=== claude-sonnet-4-5 (API, loose prompt) ===
```json
{
  "company_name": "Apple",
  "ceo_full_name": "Tim Cook",
  "headquarters_city": "Cupertino",
  "most_recent_quarter_services_revenue_usd_billion": "23.87",
  "total_fy2023_revenue_usd_billion": "383.29",
  "year_current_campus_opened": "2017",
  "approximate_worldwide_developer_attendance": "6000"
}
```


# Load the same Amazon dataset we used in the Word Vector exercise.
df = pd.read_csv("word_vector/amazon_review_large.csv")
df.columns = ["label", "text"]          # 1 = negative, 2 = positive
df["label"] = df["label"].astype(int)

# Draw 30 samples and see how LLM performs
sample = df.sample(30, random_state=42).reset_index(drop=True)
sample["gold"] = sample["label"].map({1: "NEGATIVE", 2: "POSITIVE"})
sample.head(3)


import time
from tqdm.auto import tqdm # Progress bar

SYSTEM = """Classify product reviews as POSITIVE or NEGATIVE.
Respond with exactly one word: POSITIVE or NEGATIVE. Nothing else.

Few-shot
“Not much to say it was a true Cannon battery n..” = “POSITIVE”

"""

def classify(text):
    # Note: Ensure your 'chat' function is defined to handle these arguments
    ans = chat(prompt=text, system=SYSTEM, temperature=0.0).upper()
    if "POSITIVE" in ans: return "POSITIVE"
    if "NEGATIVE" in ans: return "NEGATIVE"
    return "UNKNOWN"

predictions = []
t0 = time.time()

for i, row in tqdm(enumerate(sample.itertuples()), total=len(sample)):
    pred = classify(row.text)
    predictions.append(pred)

    if i < 15 and i % 5 == 0:
        print(f"\n--- Example {i//5 + 1} ---")
        print(f"Text: {row.text[:70]}...")
        print(f"Pred: {pred} | Gold: {row.gold}")

sample["pred"] = predictions
total_time = time.time() - t0

print(f"\n{'='*30}")
print(f"LLM prediction done in {total_time:.1f}s (Avg: {total_time/len(sample):.2f}s/row)")

acc = (sample["pred"] == sample["gold"]).mean()
unknown = (sample["pred"] == "UNKNOWN").sum()
print(f"zero-shot accuracy: {acc:.1%}   ({unknown} unparsed outputs)")
sample.head()

  0%|          | 0/30 [00:00<?, ?it/s]

--- Example 1 ---
Text: Not much to say it was a true Cannon battery not a generic battery whi...
Pred: POSITIVE | Gold: POSITIVE

--- Example 2 ---
Text: This is totally different from the "classic albums" series of DVDs tha...
Pred: NEGATIVE | Gold: NEGATIVE

--- Example 3 ---
Text: I believe the pet groomer would be better if it were smaller as it is ...
Pred: NEGATIVE | Gold: NEGATIVE

==============================
LLM prediction done in 18.2s (Avg: 0.61s/row)
zero-shot accuracy: 93.3%   (0 unparsed outputs)


from sklearn.metrics import classification_report, confusion_matrix

# Metrics need a restricted label set — drop the UNKNOWNs
clean = sample[sample["pred"] != "UNKNOWN"]
print(classification_report(clean["gold"], clean["pred"], labels=["POSITIVE","NEGATIVE"]))
print("Confusion matrix (rows=gold, cols=pred, order [POS, NEG]):")
print(confusion_matrix(clean["gold"], clean["pred"], labels=["POSITIVE","NEGATIVE"]))

              precision    recall  f1-score   support

    POSITIVE       1.00      0.87      0.93        15
    NEGATIVE       0.88      1.00      0.94        15

    accuracy                           0.93        30
   macro avg       0.94      0.93      0.93        30
weighted avg       0.94      0.93      0.93        30

Confusion matrix (rows=gold, cols=pred, order [POS, NEG]):
[[13  2]
 [ 0 15]]


ambiguous = "The product is great and functional, but the color is not what I expected."

# Run 5 times at high temperature
labels_hot = [chat(prompt=ambiguous, system=SYSTEM, temperature=1.0).strip() for _ in range(5)]
# Run 5 times at temperature 0
labels_cold = [chat(prompt=ambiguous, system=SYSTEM, temperature=0.0).strip() for _ in range(5)]

print("temperature=1.0 →", labels_hot)
print("temperature=0.0 →", labels_cold)

temperature=1.0 → ['NEGTIV', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE']
temperature=0.0 → ['NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE']


from collections import Counter

def classify_vote(text, n=10, temperature=0):
    labels = []
    for _ in range(n):
        ans = chat(prompt=text, system=SYSTEM, temperature=temperature).strip().upper()
        hits = re.findall(r"POSITIVE|NEGATIVE", ans)
        labels.append(hits[0] if hits else "NEGATIVE")
    return Counter(labels).most_common(1)[0][0], labels

# Try voting on 5 harder reviews
tricky = sample.sample(10, random_state=1).reset_index(drop=True)
for i, row in tricky.iterrows():
    pred, votes = classify_vote(row["text"][:300], n=5)
    print(f"gold={row['gold']:9}  majority={pred:9}  votes={votes}")

gold=POSITIVE   majority=POSITIVE   votes=['POSITIVE', 'POSITIVE', 'POSITIVE', 'POSITIVE', 'POSITIVE']
gold=POSITIVE   majority=POSITIVE   votes=['POSITIVE', 'POSITIVE', 'POSITIVE', 'POSITIVE', 'POSITIVE']
gold=NEGATIVE   majority=NEGATIVE   votes=['NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE']
gold=NEGATIVE   majority=NEGATIVE   votes=['NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE']
gold=POSITIVE   majority=NEGATIVE   votes=['NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE']
gold=NEGATIVE   majority=NEGATIVE   votes=['NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE']
gold=NEGATIVE   majority=NEGATIVE   votes=['NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE']
gold=POSITIVE   majority=POSITIVE   votes=['POSITIVE', 'POSITIVE', 'POSITIVE', 'POSITIVE', 'POSITIVE']
gold=NEGATIVE   majority=NEGATIVE   votes=['NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE']
gold=NEGATIVE   majority=NEGATIVE   votes=['NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE']


# a coherent topic (e.g. what LDA might produce on news data)
coherent = ["election", "vote", "candidate", "campaign", "poll", "democrat", "republican", "senator", "debate", "ballot"]
label1 = chat(
    prompt=f"These are the top words of a topic from LDA. Give a short 2-4 word human-readable label.\n\nWORDS: {coherent}",
    system="You label LDA topics. Output only the label, no explanation.",
    temperature=0.0,
)
print("coherent words →", label1)

coherent words → Election Politics


# an INCOHERENT topic — what if the topic model produced noise?
noisy = ["car", "recipe", "dog", "quantum", "lemon", "congress", "socket", "opera", "xerox", "pineapple"]
label2 = chat(
    prompt=f"These are the top words of a topic from LDA. Give a short 2-4 word human-readable label.\n\nWORDS: {noisy}",
    system="You label LDA topics. Output only the label, no explanation.",
    temperature=0.0,
)
print("noisy words →", label2)

noisy words → Food Recipe Book


# Cluster A: looks coherent (battery / charging complaints)
cluster_A_keywords = ["battery", "charge", "drain", "hours", "dies", "percent", "lasts", "overnight"]
cluster_A_docs = [
    "Battery only lasts about 4 hours of normal use, way less than advertised.",
    "Drains from full to 20% in a single Netflix episode. Unusable.",
    "Charge cycle takes forever and the battery dies overnight even when the phone is asleep.",
]

# Cluster B: incoherent on purpose — keywords are random AND the documents are
# from three obviously different domains (finance / puzzles / infrastructure)
cluster_B_keywords = ["car", "recipe", "dog", "quantum", "lemon", "congress", "socket", "opera", "xerox", "pineapple"]
cluster_B_docs = [
    "Quarterly earnings missed analyst expectations by 12%.",
    "She solved the cryptic crossword in under ten minutes.",
    "The bridge was closed for two days due to high winds.",
]

LABEL_SYS = "You label topic-model clusters. Output ONLY valid JSON."
LABEL_TMPL = (
    "Below are the top keywords and a few representative documents from one cluster of a topic model.\n"
    "Decide on a 2-4 word human-readable label and a one-sentence description.\n"
    "Also rate the cluster's coherence: high / medium / low.\n"
    "If the keywords and the documents do not share a clear common theme, set coherence to 'low' "
    "and say so in 'description'. It is acceptable to report low coherence — do not invent a theme.\n\n"
    "TOP KEYWORDS: {kw}\n\nREPRESENTATIVE DOCUMENTS:\n- {docs}\n\n"
    "Return JSON with keys: label, description, coherence."
)

for name, kw, docs in [("Cluster A", cluster_A_keywords, cluster_A_docs),
                       ("Cluster B", cluster_B_keywords, cluster_B_docs)]:
    out = chat(
        prompt=LABEL_TMPL.format(kw=", ".join(kw), docs="\n- ".join(docs)),
        system=LABEL_SYS, fmt="json",
    )
    print(f"=== {name} ===")
    print(out)
    print()

# Try the same prompt on a frontier model if you have an API key — bigger models
# are noticeably more willing to return coherence: 'low' on the noise cluster.

=== Cluster A ===
{
"label": "Short Battery Life",
"description": "This cluster appears to be about batteries that do not last long, with examples of devices draining quickly or dying after a short period.",
"coherence": "high"
}

=== Cluster B ===
{
"label": "Miscellaneous News",
"description": "This cluster appears to be a collection of unrelated news articles.",
"coherence": "low"
}


review = ("The phone battery lasts all day which I love, but the camera is truly awful "
          "in low light, and the price is ridiculous for what you get.")

# Without format enforcement
free = chat(
    prompt=f"Extract aspect-sentiment pairs from this review:\n{review}",
    system="Return each aspect with its sentiment (positive/negative/neutral).",
    temperature=0.0,
)
print("FREE FORM:\n", free, "\n")

# With JSON mode
structured = chat(
    prompt=(f"Extract aspect-sentiment pairs from this review as JSON with key "
            f"'aspects' containing a list of objects {{aspect, sentiment}}.\n\n{review}"),
    system="Output ONLY valid JSON.",
    temperature=0.0,
    fmt="json",
)
print("STRUCTURED:\n", structured)
parsed = json.loads(structured)
parsed

FREE FORM:
 Here are the aspect-sentiment pairs extracted from the review:

1. Phone battery - Positive (lasts all day)
2. Camera - Negative (awful in low light)
3. Price - Negative (ridiculous)

Note that there are only three aspects mentioned in the review, and each has a sentiment associated with it. 

STRUCTURED:
 {
"aspects": [
  {"aspect": "battery", "sentiment": "positive"},
  {"aspect": "camera", "sentiment": "negative"},
  {"aspect": "price", "sentiment": "negative"}
]
}

{'aspects': [{'aspect': 'battery', 'sentiment': 'positive'},
  {'aspect': 'camera', 'sentiment': 'negative'},
  {'aspect': 'price', 'sentiment': 'negative'}]}


implicit_review = (
    "I bought this for hiking. It's a beast — it barely fits in the top compartment of my backpack "
    "and my shoulders feel it after the first mile. The shell takes a downpour like it's nothing, "
    "but I've already had two zippers fail in three months. Customer service replaced one zipper "
    "no questions asked, the other one I'm still arguing about."
)

implicit_prompt = (
    "Extract aspect-sentiment pairs from the product review. "
    "Aspect names should reflect what the customer is actually talking about, "
    "even if they don't use that exact word. "
    "Return JSON: {aspects: [{aspect, sentiment, evidence}]}\n\n"
    f"REVIEW:\n{implicit_review}"
)

raw = chat(prompt=implicit_prompt, system="Output ONLY valid JSON.", fmt="json")
parsed = json.loads(raw)
for a in parsed.get("aspects", []):
    print(f"  {a.get('aspect','?'):22s} {a.get('sentiment','?'):10s}  '{(a.get('evidence') or '')[:60]}'")

  packability            negative    'it barely fits in the top compartment of my backpack'
  comfort                negative    'my shoulders feel it after the first mile'
  water resistance       positive    'The shell takes a downpour like it's nothing'
  durability             negative    'I've already had two zippers fail in three months'
  customer service       mixed       'Customer service replaced one zipper no questions asked, the'


docs = [
    "The laptop battery died within an hour.",
    "Excellent camera quality in low light.",
    "Ships broke down on arrival, totally unusable.",
    "Food at this restaurant was bland and overpriced.",
    "Amazing service, the staff were so helpful.",
    "The hotel room smelled like smoke.",
    "This phone drains from 100% to 20% after a Netflix episode.",
]
query = "portable power ran out too fast"
print(f"QUERY: {query}\n")

QUERY: portable power ran out too fast


# --- Method 1: TFIDF cosine (keyword overlap) ---
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

v = TfidfVectorizer().fit(docs + [query])
qv, dv = v.transform([query]), v.transform(docs)
tfidf_scores = cosine_similarity(qv, dv)[0]

print("=== TFIDF (keyword overlap) ===")
for s, d in sorted(zip(tfidf_scores, docs), reverse=True):
    print(f"  {s:.3f}  {d}")
print("\nNote how most scores are exactly 0.000 — zero keyword overlap → zero signal.")

=== TFIDF (keyword overlap) ===
  0.000  This phone drains from 100% to 20% after a Netflix episode.
  0.000  The laptop battery died within an hour.
  0.000  The hotel room smelled like smoke.
  0.000  Ships broke down on arrival, totally unusable.
  0.000  Food at this restaurant was bland and overpriced.
  0.000  Excellent camera quality in low light.
  0.000  Amazing service, the staff were so helpful.

Note how most scores are exactly 0.000 — zero keyword overlap → zero signal.


# --- Method 2: Pretrained sentence-transformer (BERT-era) ---
# pip install sentence-transformers  (~100 MB model, downloads once)
try:
    from sentence_transformers import SentenceTransformer
    st = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
    qv_st = st.encode(query)
    dv_st = st.encode(docs)
    st_scores = cosine_similarity([qv_st], dv_st)[0]
    print("=== Pretrained MiniLM (2019-era sentence encoder) ===")
    for s, d in sorted(zip(st_scores, docs), reverse=True):
        print(f"  {s:.3f}  {d}")
except Exception as e:
    print(f"[skip sentence-transformers] {e}")
    st_scores = None

README.md: 0.00B [00:00, ?B/s]

=== Pretrained MiniLM (2019-era sentence encoder) ===
  0.379  The laptop battery died within an hour.
  0.237  Ships broke down on arrival, totally unusable.
  0.195  This phone drains from 100% to 20% after a Netflix episode.
  0.106  The hotel room smelled like smoke.
  0.075  Food at this restaurant was bland and overpriced.
  0.061  Amazing service, the staff were so helpful.
  0.012  Excellent camera quality in low light.


# --- Method 3: Modern LLM-era embedding via Ollama ---
# Requires: ollama pull nomic-embed-text
def embed(text):
    return ollama.embeddings(model="nomic-embed-text", prompt=text)["embedding"]

def cos(a, b):
    a, b = np.array(a), np.array(b)
    return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))

qvec = embed(query)
ollama_scores = [cos(qvec, embed(d)) for d in docs]

print("=== Ollama nomic-embed-text (LLM-era encoder) ===")
for s, d in sorted(zip(ollama_scores, docs), reverse=True):
    print(f"  {s:.3f}  {d}")

=== Ollama nomic-embed-text (LLM-era encoder) ===
  0.648  The laptop battery died within an hour.
  0.520  This phone drains from 100% to 20% after a Netflix episode.
  0.485  Ships broke down on arrival, totally unusable.
  0.458  Excellent camera quality in low light.
  0.422  Amazing service, the staff were so helpful.
  0.415  Food at this restaurant was bland and overpriced.
  0.412  The hotel room smelled like smoke.


# --- Side-by-side ranks ---
import pandas as pd
cmp = pd.DataFrame({"doc": docs, "tfidf": tfidf_scores, "ollama": ollama_scores})
if st_scores is not None:
    cmp["miniLM"] = st_scores
cmp = cmp.set_index("doc").round(3)
cmp["tfidf_rank"]  = cmp["tfidf"].rank(ascending=False).astype(int)
cmp["ollama_rank"] = cmp["ollama"].rank(ascending=False).astype(int)
if "miniLM" in cmp.columns:
    cmp["miniLM_rank"] = cmp["miniLM"].rank(ascending=False).astype(int)
cmp


sarcastic = [
    # Overt sarcasm
    "Oh GREAT, another software update that crashed my laptop. Exactly what I needed before my presentation.",
    "Wow, a 30-minute wait on hold — truly a premium customer experience.",
    # Subtler: understatement, hedging, ironic praise
    "The movie was so incredibly detailed that I managed to finish my entire 400-page book during the opening credits.",
    "I love how the new update removed the only feature I actually used.",
    "The hotel met my expectations. My expectations were rock bottom.",
]
print("=== llama3 (local, ~8B params) ===")
for s in sarcastic:
    print(f"  [{classify(s):9}]  {s}")

=== llama3 (local, ~8B params) ===
  [NEGATIVE ]  Oh GREAT, another software update that crashed my laptop. Exactly what I needed before my presentation.
  [NEGATIVE ]  Wow, a 30-minute wait on hold — truly a premium customer experience.
  [NEGATIVE ]  The movie was so incredibly detailed that I managed to finish my entire 400-page book during the opening credits.
  [NEGATIVE ]  I love how the new update removed the only feature I actually used.
  [NEGATIVE ]  The hotel met my expectations. My expectations were rock bottom.


# Same 5 reviews, but via a larger API model. Skip if no key.
def classify_api(text, provider="anthropic"):
    try:
        if provider == "anthropic":
            from anthropic import Anthropic
            c = Anthropic(api_key=load_api_key("key/anthropic_key.txt"))
            msg = c.messages.create(
                model="claude-sonnet-4-5",
                max_tokens=10,
                system=SYSTEM,
                messages=[{"role":"user","content":text}],
            )
            ans = msg.content[0].text.upper()
        elif provider == "openai":
            from openai import OpenAI
            c = OpenAI(api_key=load_api_key("key/openai_key.txt"))
            r = c.chat.completions.create(
                model="gpt-4o-mini",
                messages=[{"role":"system","content":SYSTEM},{"role":"user","content":text}],
                temperature=0.0,
            )
            ans = r.choices[0].message.content.upper()
        else:
            return "SKIP"
        if "POSITIVE" in ans: return "POSITIVE"
        if "NEGATIVE" in ans: return "NEGATIVE"
        return "UNKNOWN"
    except Exception as e:
        return f"SKIP ({type(e).__name__})"

print("=== claude-sonnet-4-5 (via API) ===")
for s in sarcastic:
    print(f"  [{classify_api(s, 'anthropic'):15}]  {s}")

=== claude-sonnet-4-5 (via API) ===
  [NEGATIVE       ]  Oh GREAT, another software update that crashed my laptop. Exactly what I needed before my presentation.
  [NEGATIVE       ]  Wow, a 30-minute wait on hold — truly a premium customer experience.
  [NEGATIVE       ]  The movie was so incredibly detailed that I managed to finish my entire 400-page book during the opening credits.
  [NEGATIVE       ]  I love how the new update removed the only feature I actually used.
  [NEGATIVE       ]  The hotel met my expectations. My expectations were rock bottom.


tricky = "The movie was so incredibly detailed that I managed to finish my entire 400-page book during the opening credits."

labels = [classify(tricky) for _ in range(5)]
print("5 runs:", labels)
print("majority:", Counter(labels).most_common(1)[0][0])
print("true label: NEGATIVE")

5 runs: ['NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE', 'NEGATIVE']
majority: NEGATIVE
true label: NEGATIVE


reviews = {
    "rave"  : ("Best purchase of the year. I've recommended it to literally everyone I know. "
               "Wish I'd bought it sooner."),
    "mixed" : ("Mixed feelings about these $200 wireless headphones after a month of daily use. "
               "Sound quality is surprisingly good for the price. Build feels cheap — one earcup hinge "
               "already cracked. Battery is about 22 hours, advertised 30. Customer service replaced "
               "the earcup the next day, no complaints. Overall worth $150, not $200. Recommend only on sale."),
    "rant"  : "Total garbage. Stopped working after a week. Support never replied. Avoid.",
}

# --- Test 1: vague rubric vs. anchored NPS rubric ---
vague_system  = (
    "Rate this customer review on a 0-10 scale (higher = more positive). "
    "Return ONLY the integer, nothing else."
)
anchored_system = (
    "Rate how likely this reviewer is to recommend the product on a 0-10 scale.\n"
    "Use this rubric:\n"
    "  0-3 : would actively warn others away (detractor)\n"
    "  4-6 : indifferent or mixed (passive)\n"
    "  7-8 : would casually recommend (promoter)\n"
    "  9-10: would enthusiastically promote (strong promoter)\n"
    "Return ONLY the integer."
)

def score_n(text, sys, n=5, temperature=0.0):
    out = []
    for _ in range(n):
        r = chat(prompt=text, system=sys, temperature=temperature).strip()
        m = re.search(r"\b(10|[0-9])\b", r)
        out.append(int(m.group()) if m else None)
    return out

print(f"{'review':6}  {'vague rubric (5 runs)':28}  {'anchored rubric (5 runs)':28}")
print("-" * 70)
for name, text in reviews.items():
    v = score_n(text, vague_system, n=5)
    a = score_n(text, anchored_system, n=5)
    print(f"{name:6}  {str(v):28}  {str(a):28}")

review  vague rubric (5 runs)         anchored rubric (5 runs)    
----------------------------------------------------------------------
rave    [10, 10, 10, 10, 10]          [10, 10, 10, 10, 10]        
mixed   [6, 6, 6, 6, 6]               [7, 7, 7, 7, 7]             
rant    [1, 1, 1, 1, 1]               [0, 0, 0, 0, 0]


lenient_system = (
    "You are a *lenient* reviewer. You believe most products are doing their best and you "
    "give the benefit of the doubt; small flaws don't bother you. "
    "Rate this customer review on a 0-10 scale (higher = more positive). Return ONLY the integer."
)
strict_system = (
    "You are a *strict* reviewer. You deduct points aggressively for any flaw, broken feature, "
    "missed advertised spec, or sign of poor build quality. "
    "Rate this customer review on a 0-10 scale (higher = more positive). Return ONLY the integer."
)

print(f"{'review':6}  {'lenient persona':28}  {'strict persona':28}  {'gap':>5}")
print("-" * 80)
for name, text in reviews.items():
    L = score_n(text, lenient_system, n=5)
    S = score_n(text, strict_system, n=5)
    gap = (sum(L)/len(L)) - (sum(S)/len(S))
    print(f"{name:6}  {str(L):28}  {str(S):28}  {gap:+.1f}")

review  lenient persona               strict persona                  gap
--------------------------------------------------------------------------------
rave    [10, 10, 10, 10, 10]          [9, 9, 9, 9, 9]               +1.0
mixed   [7, 7, 7, 7, 7]               [4, 4, 4, 4, 4]               +3.0
rant    [2, 2, 2, 2, 2]               [1, 1, 1, 1, 1]               +1.0


# ---------- Accuracy + Reliability + Agreement ----------
from sklearn.metrics import f1_score, cohen_kappa_score

clean = sample[sample["pred"] != "UNKNOWN"]

# (1) ACCURACY — LLM vs. human gold.
f1_pos    = f1_score(clean["gold"], clean["pred"], pos_label="POSITIVE")
print(f"[accuracy]    F1 (positive class)      : {f1_pos:.3f}")

# (2) RELIABILITY — same prompt, second pass. High kappa = stable, NOT necessarily correct.
run2 = sample["text"].apply(classify)
both_ok = (sample["pred"] != "UNKNOWN") & (run2 != "UNKNOWN")
kappa_self = cohen_kappa_score(sample["pred"][both_ok], run2[both_ok])
print(f"[reliability] κ (run1 vs. run2)         : {kappa_self:.3f}   (1.0 = identical, 0 = random)")

# (3) INTER-RATER AGREEMENT — LLM vs. human gold.
kappa_human = cohen_kappa_score(clean["gold"], clean["pred"])
print(f"[agreement]   κ (LLM vs. human gold)   : {kappa_human:.3f}")

[accuracy]    F1 (positive class)      : 0.929
[reliability] κ (run1 vs. run2)         : 1.000   (1.0 = identical, 0 = random)
[agreement]   κ (LLM vs. human gold)   : 0.867


# ---------- Robustness + LLM-as-judge ----------
# (4) ROBUSTNESS — do small rewrites of the prompt change the labels?
SYS_V1 = "Classify product reviews as POSITIVE or NEGATIVE. Respond with one word."
SYS_V2 = "Decide if the customer is happy with their purchase. Output exactly POSITIVE or NEGATIVE."
SYS_V3 = "Is the following review positive or negative overall? Reply POSITIVE or NEGATIVE."

def classify_with(system_prompt, text):
    ans = chat(prompt=text, system=system_prompt, temperature=0.0).upper()
    if "POSITIVE" in ans: return "POSITIVE"
    if "NEGATIVE" in ans: return "NEGATIVE"
    return "UNKNOWN"

sub = sample.head(15)
v1 = sub["text"].apply(lambda t: classify_with(SYS_V1, t))
v2 = sub["text"].apply(lambda t: classify_with(SYS_V2, t))
v3 = sub["text"].apply(lambda t: classify_with(SYS_V3, t))
mask = (v1 != "UNKNOWN") & (v2 != "UNKNOWN") & (v3 != "UNKNOWN")
print(f"[robustness]  κ (prompt v1 vs. v2)     : {cohen_kappa_score(v1[mask], v2[mask]):.3f}")
print(f"[robustness]  κ (prompt v1 vs. v3)     : {cohen_kappa_score(v1[mask], v3[mask]):.3f}")

# (5) LLM-AS-JUDGE — have an LLM grade the classifier's output on a Likert scale.
# In real use a *stronger* model judges a weaker one; here we re-use llama3 for demo.
JUDGE = (
    "You are evaluating a sentiment classifier. The classifier was asked whether a product "
    "review is POSITIVE or NEGATIVE. On a Likert scale of 1-5 (1 = clearly wrong, "
    "5 = clearly correct), how good is the classifier's answer? Reply with only the integer."
)

def judge_one(review, label):
    r = chat(prompt=f"REVIEW:\n{review}\n\nCLASSIFIER ANSWER: {label}",
             system=JUDGE, temperature=0.0).strip()
    m = re.search(r"\b[1-5]\b", r)
    return int(m.group()) if m else None

scores = [judge_one(t, p) for t, p in zip(clean["text"].head(8), clean["pred"].head(8))]
scores = [s for s in scores if s is not None]
print(f"[llm-judge]   mean Likert (1-5) over {len(scores)} items: {np.mean(scores):.2f}")

[robustness]  κ (prompt v1 vs. v2)     : 0.851
[robustness]  κ (prompt v1 vs. v3)     : 1.000
[llm-judge]   mean Likert (1-5) over 8 items: 4.88

Provider	What they make	Cost to try	Live docs (pricing + current models)
OpenAI	GPT-4o, GPT-5, o-series	paid (trial credit)	developers.openai.com/api/docs/models
Anthropic	Claude Haiku / Sonnet / Opus	paid (trial credit)	platform.claude.com/docs/.../models/overview
Google	Gemini 2.5 Flash / Pro	free tier	ai.google.dev/gemini-api/docs/models
Ollama	runs open-weight models locally (Llama, Qwen, Mistral, …)	free, no key, offline	ollama.com/library

Param	What it does
`temperature`	randomness — 0 = deterministic, 1+ = creative
`top_p`	nucleus sampling (alternative to temperature)
`max_tokens` / `max_gen_len`	cap the output length
`stop`	strings that end generation
`stream`	return tokens as they're generated
`seed`	best-effort reproducibility

Step	Tool of choice	Why
Group documents into coherent clusters	LDA / k-means / HDBSCAN / BERTopic	deterministic, scalable, gives you a stable cluster ID
Inspect the top keywords or representative documents per cluster	human analyst	sanity check
Label and describe each cluster	LLM	this is the bottleneck a human used to do by hand

Method	Era	How it works	When it fails
TFIDF cosine	2000s	counts shared words, weighted by rarity	returns 0 when no word overlap
Pretrained sentence-transformer (MiniLM)	2019	small BERT-style encoder, 384-dim	slower than TFIDF; domain-specific jargon
Modern LLM-era embedding	2020+	larger encoder trained with LLM techniques, 768-dim	bigger download, slightly slower

	TF-IDF	Sentence-transformer (MiniLM, 2019)	LLM-era embedding (`nomic-embed-text`, 2024)
What it captures	term overlap, weighted by rarity	sentence-level meaning, learned from sentence-pair contrast	sentence + paragraph meaning, learned at LLM scale
Vector	sparse, ~vocab-size	dense, 384	dense, 768
Context window	n/a	~256 tokens	up to 8192 tokens — fits a full document
Training data	none, just statistics on your corpus	~1B sentence pairs, English-leaning	trillions of tokens, multilingual, modern web
Cost	free, instant	~100 MB model, runs on CPU	~270 MB model, runs on CPU; or paid API call

BIA 660 Spring 2026 — Introduction to Large Language Models (LLMs)¶

§0 Local LLM Setup¶

§1 Try on different models — APIs vs. local open-source¶

1.1 OpenAI¶

1.2 Anthropic (Claude)¶

1.3 Google (Gemini)¶

1.4 Ollama — local, free¶

1.5 A brief note on parameters¶

§2 Use LLMs to perform different textual analysis¶

2.1 Text preprocessing & NER — replaces hand-rolled NLTK pipelines¶

2.2 Classification — zero-shot vs. supervised classifiers¶

2.3 Reproducibility check¶

2.4 Majority voting — the easy reliability upgrade¶

2.5 Topic labeling — interpretation, not clustering¶

2.6 Aspect-based sentiment with structured output¶

2.7 Embeddings & semantic search — replaces TFIDF cosine¶

§3 Where LLMs struggle — the subtle text-analysis cases¶

3.1 Sarcasm & irony¶

3.2 Self-consistency ≠ correctness¶

3.3 Why fine-grained numerical ratings are unreliable¶

§4 Evaluation toolkit — how do you know the LLM did a good job?¶

§5 Summary: When to use an LLM for text analysis¶

What this means in practice¶

Suggested workflow to use LLM in your project¶

	label	text	gold
0	2	Not much to say it was a true Cannon battery n...	POSITIVE
1	1	I do not recommend this book is so unsubstanti...	NEGATIVE
2	2	This isn't a movie that's afraid to be itself....	POSITIVE

	tfidf	ollama	miniLM	tfidf_rank	ollama_rank	miniLM_rank
doc
The laptop battery died within an hour.	0.0	0.648	0.379	4	1	1
Excellent camera quality in low light.	0.0	0.458	0.012	4	4	7
Ships broke down on arrival, totally unusable.	0.0	0.485	0.237	4	3	2
Food at this restaurant was bland and overpriced.	0.0	0.415	0.075	4	6	5
Amazing service, the staff were so helpful.	0.0	0.422	0.061	4	5	6
The hotel room smelled like smoke.	0.0	0.412	0.106	4	7	4
This phone drains from 100% to 20% after a Netflix episode.	0.0	0.520	0.195	4	2	3

Evaluation	What it answers	Method
Accuracy	Is the LLM right?	Compare LLM output to human gold labels: `accuracy`, `f1`, `classification_report`, `confusion_matrix`.
Reliability	Does the LLM agree with itself across runs?	Run N times at temperature=0, take majority vote, look at vote spread.
Inter-rater agreement	Does the LLM agree with humans? Or with another LLM?	Cohen's κ between LLM labels and human labels (or between two LLMs).
Robustness	Do small prompt rewrites flip the answer?	Run the same items through 2-3 paraphrased prompts. Compute κ between the runs. Different from reliability: reliability tests the same prompt twice; robustness tests slightly different prompts.
LLM-as-judge	Use a stronger LLM to grade a weaker one's output on a Likert scale. Less standard — needs strong justification in academic work.	G-Eval / pairwise comparison / Likert (1-5 or 1-7).

Task	LLM fit	Main strength	Main pitfall
Classification with clear, well-defined classes (sentiment, topic, spam)	✅ strong	Zero/few-shot — no training data needed	Boundary cases drift; vote across runs
Named Entity Recognition & information extraction	✅ strong	Contextual, flexible schemas	May fabricate absent fields — require "null if missing"
Structured output to a schema (JSON)	✅ strong	Format enforcement makes it pipeline-ready	Vocabulary drift without examples
Summarization, paraphrase	✅ strong	Core generation capability	Can add details not in source — always check
Topic labeling from keyword lists	✅ strong	Compresses long keyword lists into readable names	Invents a confident label on incoherent input
Semantic search & similarity (embeddings)	✅ strong	Captures meaning beyond shared words	Domain shift degrades quality
Subjective or fine-grained categorization (nuanced emotion, intensity)	⚠️ moderate	Can approximate a well-anchored rubric	Humans also disagree — drift across runs (§3.3)
Tasks requiring implicit or culturally-coded context (sarcasm, irony)	⚠️ moderate	Larger frontier models improve noticeably (§3.1)	Small local models miss the subtle cases
Domain-specialty coding (legal, medical, financial)	⚠️ moderate	General reading comprehension	Pretraining exposure dependent — consider fine-tune or RAG