Skip to content

fix(examples-py): validate Chroma, Hugging Face, and Instructor OTel 2.x spans #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Nightingalelyy wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(examples-py): validate Chroma, Hugging Face, and Instructor OTel 2.x spans #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions python/tracing/chroma/_shared.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,15 @@ def load_example_env() -> None:

def create_respan(workflow_name: str) -> Respan:
load_example_env()
run_id = os.getenv("RESPAN_EXAMPLE_RUN_ID") or uuid4().hex[:8]
return Respan(
api_key=os.environ["RESPAN_API_KEY"],
base_url=os.getenv("RESPAN_BASE_URL", RESPAN_BASE_URL),
app_name=workflow_name,
metadata={
"example_set": EXAMPLE_SET,
"workflow_name": workflow_name,
"run_id": run_id,
},
instrumentations=[ChromaInstrumentor()],
is_batching_enabled=False,
Expand All @@ -82,6 +84,7 @@ def workflow_attributes(workflow_name: str) -> dict[str, object]:
"example_set": EXAMPLE_SET,
"workflow_name": workflow_name,
"example_run_id": run_id,
"run_id": run_id,
},
}

Expand Down
87 changes: 87 additions & 0 deletions python/tracing/huggingface/04_real_tiny_pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
"""Trace an actual local Transformers pipeline without downloading a model."""

from __future__ import annotations

import torch
from respan import workflow
from tokenizers import Tokenizer
from tokenizers.models import WordLevel
from tokenizers.pre_tokenizers import Whitespace
from transformers import (
GPT2Config,
GPT2LMHeadModel,
PreTrainedTokenizerFast,
TextGenerationPipeline,
)

from _shared import build_respan, print_result

EXAMPLE_NAME = "real-tiny-pipeline"
WORKFLOW_NAME = "huggingface_04_real_tiny_pipeline"


def build_pipeline() -> TextGenerationPipeline:
vocabulary = {
"[PAD]": 0,
"[UNK]": 1,
"[EOS]": 2,
"Tracing": 3,
"Hugging": 4,
"Face": 5,
"locally": 6,
"keeps": 7,
"telemetry": 8,
"repeatable": 9,
".": 10,
}
tokenizer_backend = Tokenizer(
WordLevel(vocab=vocabulary, unk_token="[UNK]")
)
tokenizer_backend.pre_tokenizer = Whitespace()
tokenizer = PreTrainedTokenizerFast(
tokenizer_object=tokenizer_backend,
unk_token="[UNK]",
pad_token="[PAD]",
eos_token="[EOS]",
)

torch.manual_seed(17)
model = GPT2LMHeadModel(
GPT2Config(
vocab_size=len(vocabulary),
name_or_path="respan-local-tiny-gpt2",
n_positions=32,
n_ctx=32,
n_embd=16,
n_layer=1,
n_head=1,
bos_token_id=2,
eos_token_id=2,
pad_token_id=0,
)
)
model.eval()
return TextGenerationPipeline(model=model, tokenizer=tokenizer, device=-1)


@workflow(name=WORKFLOW_NAME)
def execute_real_tiny_generation() -> list[dict[str, str]]:
return build_pipeline()(
"Tracing Hugging Face locally",
max_new_tokens=4,
do_sample=False,
)


def run() -> list[dict[str, str]]:
respan = build_respan(example_name=EXAMPLE_NAME, workflow_name=WORKFLOW_NAME)
try:
result = execute_real_tiny_generation()
print_result("real_tiny_generation", result)
return result
finally:
respan.shutdown()


if __name__ == "__main__":
run()
1 change: 1 addition & 0 deletions python/tracing/huggingface/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,4 @@ python run_all.py
| `01_text_generation_pipeline.py` | `huggingface_01_text_generation_pipeline` | Single `TextGenerationPipeline.__call__`, model metadata, generation parameters, prompt, and completion |
| `02_batch_prompts.py` | `huggingface_02_batch_prompts` | Batch prompt indexing and multiple generated completions |
| `03_trace_content_disabled.py` | `huggingface_03_trace_content_disabled` | `TRACELOOP_TRACE_CONTENT=false` privacy mode with metadata-only generation spans |
| `04_real_tiny_pipeline.py` | `huggingface_04_real_tiny_pipeline` | Real local `transformers.TextGenerationPipeline` with an in-memory tokenizer and randomly initialized tiny GPT-2 model; no model download or provider credential |
9 changes: 6 additions & 3 deletions python/tracing/huggingface/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
python-dotenv
respan-ai
respan-instrumentation-huggingface
opentelemetry-instrumentation-transformers>=0.61.0
python-dotenv>=1.0.0
respan-ai>=2.17.0
respan-instrumentation-huggingface>=0.1.0
torch>=2.2.0
transformers>=4.0
1 change: 1 addition & 0 deletions python/tracing/huggingface/run_all.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"01_text_generation_pipeline.py",
"02_batch_prompts.py",
"03_trace_content_disabled.py",
"04_real_tiny_pipeline.py",
]


Expand Down
25 changes: 13 additions & 12 deletions python/tracing/instructor/01_create.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@

from typing import TypedDict

from _respan_instructor import create_respan_instructor_client
from respan_tracing import workflow
from respan_tracing.exporters import propagate_attributes

from _respan_instructor import create_respan_instructor_client


class InvoiceSummary(TypedDict):
vendor: str
Expand All @@ -19,7 +18,7 @@ class InvoiceSummary(TypedDict):


@workflow(name="instructor_example_01_create")
def extract_invoice(client) -> InvoiceSummary:
def extract_invoice(client, scenario: str) -> InvoiceSummary:
return client.create(
response_model=InvoiceSummary,
messages=[
Expand All @@ -37,15 +36,17 @@ def extract_invoice(client) -> InvoiceSummary:


def run_create_example() -> None:
telemetry, client = create_respan_instructor_client(app_name="instructor-create")

with propagate_attributes(
thread_identifier="instructor_example_01_create",
metadata={"example_script": "01_create.py", "instructor_api": "create"},
):
invoice = extract_invoice(client)

print(dict(invoice))
respan, client = create_respan_instructor_client(app_name="instructor-create")
try:
with propagate_attributes(
thread_identifier="instructor_example_01_create",
metadata={"example_script": "01_create.py", "instructor_api": "create"},
):
invoice = extract_invoice(client, "extract a deterministic invoice")

print(dict(invoice))
finally:
respan.shutdown()


if __name__ == "__main__":
Expand Down
44 changes: 24 additions & 20 deletions python/tracing/instructor/02_validation_hooks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,13 @@

from __future__ import annotations

from typing import Literal
from typing import TypedDict
from typing import Literal, TypedDict

from _respan_instructor import create_respan_instructor_client
from instructor.core.hooks import HookName
from respan_tracing import workflow
from respan_tracing.exporters import propagate_attributes

from _respan_instructor import create_respan_instructor_client


class SupportEscalation(TypedDict):
customer: str
Expand All @@ -21,7 +19,7 @@ class SupportEscalation(TypedDict):


@workflow(name="instructor_example_02_validation_hooks")
def classify_support_escalation(client) -> SupportEscalation:
def classify_support_escalation(client, scenario: str) -> SupportEscalation:
return client.create(
response_model=SupportEscalation,
max_retries=2,
Expand All @@ -39,7 +37,7 @@ def classify_support_escalation(client) -> SupportEscalation:


def run_validation_hooks_example() -> None:
telemetry, client = create_respan_instructor_client(
respan, client = create_respan_instructor_client(
app_name="instructor-validation-hooks"
)
hook_counts = {"completion_kwargs": 0, "completion_response": 0}
Expand All @@ -61,21 +59,27 @@ def on_completion_response(_response) -> None:
client.on(HookName.COMPLETION_RESPONSE, on_completion_response)

try:
with propagate_attributes(
customer_identifier="customer_instructor_example",
thread_identifier="instructor_example_02_validation_hooks",
metadata={
"example_script": "02_validation_hooks.py",
"instructor_api": "create_hooks",
},
):
escalation = classify_support_escalation(client)
try:
with propagate_attributes(
customer_identifier="customer_instructor_example",
thread_identifier="instructor_example_02_validation_hooks",
metadata={
"example_script": "02_validation_hooks.py",
"instructor_api": "create_hooks",
},
):
escalation = classify_support_escalation(
client,
"classify a deterministic support escalation",
)
finally:
client.off(HookName.COMPLETION_KWARGS, on_completion_kwargs)
client.off(HookName.COMPLETION_RESPONSE, on_completion_response)

print(dict(escalation))
print(hook_counts)
finally:
client.off(HookName.COMPLETION_KWARGS, on_completion_kwargs)
client.off(HookName.COMPLETION_RESPONSE, on_completion_response)

print(dict(escalation))
print(hook_counts)
respan.shutdown()


if __name__ == "__main__":
Expand Down
53 changes: 28 additions & 25 deletions python/tracing/instructor/03_create_with_completion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
"""Return the parsed model and the raw provider completion."""
"""Return the parsed model and a bounded provider completion summary."""

from __future__ import annotations

from typing import Literal
from typing import TypedDict
from typing import Literal, TypedDict

from _respan_instructor import create_respan_instructor_client
from respan_tracing import workflow
from respan_tracing.exporters import propagate_attributes

from _respan_instructor import create_respan_instructor_client


class ReleaseNote(TypedDict):
title: str
Expand All @@ -19,8 +17,8 @@ class ReleaseNote(TypedDict):


@workflow(name="instructor_example_03_create_with_completion")
def draft_release_note(client) -> tuple[ReleaseNote, object]:
return client.create_with_completion(
def draft_release_note(client, scenario: str) -> tuple[ReleaseNote, dict[str, object]]:
release_note, completion = client.create_with_completion(
response_model=ReleaseNote,
messages=[
{
Expand All @@ -33,30 +31,35 @@ def draft_release_note(client) -> tuple[ReleaseNote, object]:
}
],
)
return release_note, {
"completion_type": type(completion).__name__,
"completion_id": getattr(completion, "id", None),
"completion_model": getattr(completion, "model", None),
}


def run_create_with_completion_example() -> None:
telemetry, client = create_respan_instructor_client(
respan, client = create_respan_instructor_client(
app_name="instructor-create-with-completion"
)

with propagate_attributes(
thread_identifier="instructor_example_03_create_with_completion",
metadata={
"example_script": "03_create_with_completion.py",
"instructor_api": "create_with_completion",
},
):
release_note, completion = draft_release_note(client)

print(dict(release_note))
print(
{
"completion_type": type(completion).__name__,
"completion_id": getattr(completion, "id", None),
"completion_model": getattr(completion, "model", None),
}
)
try:
with propagate_attributes(
thread_identifier="instructor_example_03_create_with_completion",
metadata={
"example_script": "03_create_with_completion.py",
"instructor_api": "create_with_completion",
},
):
release_note, completion_summary = draft_release_note(
client,
"draft a deterministic release note",
)

print(dict(release_note))
print(completion_summary)
finally:
respan.shutdown()


if __name__ == "__main__":
Expand Down
Loading