LangSmith Cheatsheet: Tracing, Datasets and Evaluation

# SDK + tracing helpers
pip install -U langsmith
pip install -U langchain                         # optional — auto-traced when LANGSMITH_TRACING=true

# env — the only required vars
export LANGSMITH_API_KEY=ls-...
export LANGSMITH_TRACING=true                    # auto-trace LangChain / LangGraph
export LANGSMITH_PROJECT="my-app-prod"           # bucket traces by env / version
export LANGSMITH_ENDPOINT=https://api.smith.langchain.com   # self-host -> your URL

# Quick connectivity check
python - <<'PY'
from langsmith import Client
print(Client().info)                              # 200 + tenant info if wired up
PY

from langsmith import traceable, get_current_run_tree
from openai import OpenAI

client = OpenAI()

@traceable(name="summarise", tags=["v2"], metadata={"model": "gpt-4o-mini"})
def summarise(text: str) -> str:
    """One traced unit of work. Nested @traceable calls become child runs."""
    resp = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[
            {"role": "system", "content": "Summarise in one sentence."},
            {"role": "user",   "content": text},
        ],
    )
    # Attach extra signal to the current run for filtering in the UI
    run = get_current_run_tree()
    run.add_metadata({"input_chars": len(text)})
    return resp.choices[0].message.content

# Wrap an entire OpenAI client so EVERY call becomes a child run
from langsmith.wrappers import wrap_openai
client = wrap_openai(OpenAI())             # now untraceable calls are traced too

from langsmith import Client
from langsmith.evaluation import evaluate, LangChainStringEvaluator

ls = Client()

# 1 · Build / fetch a dataset
ds = ls.create_dataset("faq-golden", description="Hand-curated FAQ Q&A")
ls.create_examples(
    inputs=[{"question": "How do I reset my password?"}],
    outputs=[{"answer":   "Go to Settings > Security."}],
    dataset_id=ds.id,
)

# 2 · Define the system under test
def target(inputs: dict) -> dict:
    return {"answer": my_chain.invoke(inputs["question"])}

# 3 · Custom evaluator (any callable returning {key, score, comment?})
def exact_match(run, example) -> dict:
    return {
        "key":   "exact_match",
        "score": run.outputs["answer"].strip() == example.outputs["answer"].strip(),
    }

# 4 · Run the evaluation — UI gets a comparison view
results = evaluate(
    target,
    data="faq-golden",
    evaluators=[exact_match, LangChainStringEvaluator("qa")],
    experiment_prefix="v2-prompt",
    max_concurrency=8,
)
print(results.experiment_name)

# Trace, attach feedback, run an eval against a versioned prompt — end to end.
import os
from langsmith import Client, traceable
from langsmith.evaluation import evaluate
from langsmith.wrappers import wrap_openai
from openai import OpenAI

os.environ.setdefault("LANGSMITH_TRACING", "true")
os.environ.setdefault("LANGSMITH_PROJECT", "support-bot")

ls = Client()
oai = wrap_openai(OpenAI())                     # every call is traced

# Pull a prompt from the hub (server-side versioned)
prompt = ls.pull_prompt("my-org/support-faq")    # ChatPromptTemplate-like

@traceable(name="answer")
def answer(question: str) -> str:
    msgs = prompt.invoke({"question": question}).to_messages()
    resp = oai.chat.completions.create(model="gpt-4o-mini", messages=msgs)
    return resp.choices[0].message.content

# Attach user feedback to the latest run (e.g. thumbs-up button)
from langsmith.run_helpers import get_current_run_tree
def thumbs_up():
    run = get_current_run_tree()
    ls.create_feedback(run.id, key="user_thumbs", score=1)

# Periodic regression eval
def target(inputs): return {"answer": answer(inputs["question"])}
def has_link(run, ex): return {"key": "has_link", "score": "http" in run.outputs["answer"]}

evaluate(target, data="faq-golden", evaluators=[has_link], experiment_prefix="nightly")